merge: retain external refresh and predecessor notifications

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 20:27:08 -05:00
commit 28f37718da
230 files changed
+9690 -1275

No files matched your search

@@ -597,8 +597,10 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => {
await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: true, intervalSec: 600 } } });
await vi.advanceTimersByTimeAsync(2000);
expect(idleWorkSnapshot().active).toBe(1);
// The producer committed: retain its resume deadline without keeping the
// instance busy throughout the drain. Uncertain writes still hold admission.
expect(idleWorkSnapshot().active).toBe(0);
expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now());
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction");
try {
await vi.advanceTimersByTimeAsync(60_000);
@@ -608,6 +610,11 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => {
stopTaskDrain();
await vi.advanceTimersByTimeAsync(1000);
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(now.getTime() + 600_000), { timeout: 3000 });
await vi.advanceTimersByTimeAsync(coordinator!.nextWakeAt()! - Date.now());
await vi.waitFor(() => expect(f.enqueueWakeup).toHaveBeenCalledOnce());
expect(f.enqueueWakeup).toHaveBeenCalledWith(f.agent.id, expect.objectContaining({ source: "timer", reason: "heartbeat_timer" }));
await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: false } } });
await quiet();
});
it("does not inspect timer rows during startup drain, then reconstructs them on resume", async () => {
@@ -0,0 +1,897 @@
import { randomUUID } from "node:crypto";
import { and, eq, sql, type SQL } from "drizzle-orm";
import { PgDialect } from "drizzle-orm/pg-core";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox,
decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions,
issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db";
import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js";
import { budgetService } from "../services/budgets.js";
import { writeExecutionControl } from "../services/execution-control-notifications.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { attentionService } from "../services/attention.js";
import { companyService } from "../services/companies.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js";
import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js";
import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../services/recovery/origins.js";
import { issueThreadInteractionService } from "../services/issue-thread-interactions.js";
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js";
import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "../services/retention-work-notifications.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL");
const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
(support.supported ? describe : describe.skip)("remaining attention retention", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
let companyId: string, agentId: string, draining = false;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
const errors: unknown[] = [];
const statements: string[] = [];
const parameterCounts: number[] = [];
beforeAll(async () => {
if (externalUrl) await applyPendingMigrations(externalUrl);
else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-");
db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
db.$client.options.onnotice = () => {};
db.$client.options.debug = (_connection, query, parameters) => { statements.push(query); parameterCounts.push(parameters.length); };
}, 30_000);
beforeEach(async () => {
companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0;
await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `A${companyId.slice(0, 6)}` });
await db.insert(authUsers).values({ id: "board", name: "Board", email: "board@retention.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "owner", status: "active" });
await db.insert(agents).values({ id: agentId, companyId, name: "Origin", role: "engineer", status: "active" });
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true });
vi.setSystemTime(new Date("2026-11-01T12:00:00Z"));
});
afterEach(async () => {
await coordinator?.stop(); coordinator = undefined;
vi.useRealTimers(); vi.restoreAllMocks();
await db.transaction(async tx => {
await tx.execute(sql`set local client_min_messages = warning`);
await tx.execute(sql`drop trigger if exists attention_retention_fail on decision_retention`);
await tx.execute(sql`drop function if exists attention_retention_fail()`);
await tx.execute(sql`truncate companies cascade`);
});
expect(errors).toEqual([]);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
const old = () => new Date(Date.now() - delay - 600_000);
async function insertIssue(status = "todo", values: Partial<typeof issues.$inferInsert> = {}, database: Db = db) {
const [row] = await database.insert(issues).values({ companyId, title: randomUUID(), status,
createdByAgentId: agentId, createdAt: old(), updatedAt: old(), ...values }).returning();
return row;
}
async function change<T>(write: (tx: Db) => PromiseLike<T>, scope = { companyId }) {
return db.transaction(async tx => { await notifyAttentionRetentionWork(tx, scope); return write(tx as unknown as Db); });
}
const state = (kind: string, id: string) => db.select().from(decisionRetention).where(and(
eq(decisionRetention.companyId, companyId), eq(decisionRetention.sourceKind, kind), eq(decisionRetention.sourceId, id)))
.then(rows => rows[0] ?? null);
async function settle() {
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 15_000 });
}
async function quiet() {
await settle(); expect(coordinator!.nextWakeAt()).toBeNull();
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction");
statements.length = 0;
await vi.advanceTimersByTimeAsync(300_000);
expect(statements).toEqual([]);
expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled();
select.mockRestore(); execute.mockRestore(); transaction.mockRestore();
}
async function start(service = attentionRetentionService(db)) {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) });
await registerAttentionRetentionWork(coordinator, service, () => !draining).ready;
return service;
}
async function archived(kind: string, id: string) {
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect((await state(kind, id))?.archivedAt).toBeInstanceOf(Date); }, { timeout: 15_000 });
}
async function loadAll(service = attentionRetentionService(db)) {
const entries = [];
for (;;) { const page = await service.load({ companyId }); entries.push(...page.rows ?? []); if (page.complete) return entries; }
}
const mutation = (sourceKind: "review", sourceId: string) => ({ companyId, sourceKind, sourceId,
authActor: { type: "board" as const, source: "local_implicit" as const, userId: "board", companyIds: [companyId], isInstanceAdmin: true },
actor: { actorType: "user" as const, actorId: "board", userId: "board", agentId: null, runId: null, agentApiKeyId: null, responsibleUserId: "board" } });
it("records the aggregate baseline SQL across old ticks and eliminates it after settlement", async () => {
await insertIssue("done");
const ticks: number[] = [];
for (let tick = 0; tick < 3; tick++) {
statements.length = 0;
const feed = await attentionService(db).list(companyId, { includeDismissed: true, excludeSources: ["approval", "join_request"], all: true, allowUnscopedAll: true });
await decisionRetentionService(db).autoArchive({ companyId, items: feed.items });
ticks.push(statements.length);
await vi.advanceTimersByTimeAsync(30_000);
}
expect(ticks.every(count => count > 0)).toBe(true);
process.stdout.write(JSON.stringify({ baselineStatementsPer30sTick: ticks, treatmentIdle300s: 0 }) + "\n");
await start(); await quiet();
});
it("has no recurring SQL when empty or when terminal source history remains", async () => {
await insertIssue("done"); await insertIssue("cancelled");
await start(); await quiet();
});
it("wakes from the ordinary task producer, waits until due, archives with an outbox, and settles", async () => {
await start(); await quiet();
const row = await issueService(db).create(companyId, { title: "Review a delivered result", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId });
await settle();
const [saved] = await db.select({ at: sql<number>`ceil(extract(epoch from ${issues.updatedAt}) * 1000)`.mapWith(Number) }).from(issues).where(eq(issues.id, row.id));
const due = saved.at + delay;
expect(coordinator!.nextWakeAt()).toBe(due); expect(await state("review", row.id)).toBeNull();
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute");
await vi.advanceTimersByTimeAsync(due - Date.now() - 1000);
expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); select.mockRestore(); execute.mockRestore();
await vi.advanceTimersByTimeAsync(2000); await archived("review", row.id); await quiet();
const outbox = await db.select().from(decisionArchiveNotificationOutbox);
expect(outbox).toHaveLength(1); expect(outbox[0]).toMatchObject({ companyId, sourceKind: "review", sourceId: row.id, originAgentId: agentId, originIssueId: row.id });
expect((await db.select().from(issues).where(eq(issues.id, row.id)))[0].status).toBe("in_review");
await coordinator!.stop(); coordinator = undefined; await start(); await quiet();
});
it("registers the ordinary createChild blocker path inside its relation transaction", async () => {
const parent = await issueService(db).create(companyId, { title: "Parent", status: "in_progress", assigneeAgentId: agentId, createdByAgentId: agentId });
await start(); await quiet();
const { issue: child, parentBlockerAdded } = await issueService(db).createChild(parent.id, {
title: "Child needs review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId,
blockParentUntilDone: true,
});
expect(parentBlockerAdded).toBe(true);
expect(await db.select().from(issueRelations).where(and(eq(issueRelations.issueId, child.id),
eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1);
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
await vi.advanceTimersByTimeAsync(delay + 2000); await archived("review", child.id); await quiet();
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, child.id))).toHaveLength(1);
});
it("batches a full page of maintained reviews and confirmations by company", async () => {
const rows = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
title: randomUUID(), status: "in_review", assigneeAgentId: agentId, updatedAt: old() }))).returning();
await db.insert(issueThreadInteractions).values(rows.map(row => ({ companyId, issueId: row.id,
kind: "request_confirmation", addresseeAgentId: agentId, payload: { version: 1, prompt: "Agent verdict" } })));
statements.length = 0;
const page = await attentionRetentionService(db).load({ companyId, sourceKind: "review" });
expect(page.rows).toEqual([]); expect(page.complete).toBe(false);
expect(statements.length).toBeLessThanOrEqual(20);
expect(statements.filter(query => query.includes('from "agents"')).length).toBeLessThanOrEqual(1);
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId));
statements.length = 0;
const interactions = await attentionRetentionService(db).load({ companyId, sourceKind: "issue_thread_interaction" });
expect(interactions.rows).toHaveLength(100); expect(interactions.complete).toBe(false);
expect(statements.length).toBeLessThanOrEqual(6);
expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1);
});
it("batches decision and recovery source links and rechecks a late hidden origin", async () => {
const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
title: randomUUID(), status: "todo", updatedAt: old() }))).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded" }).returning();
const decisionRows = await db.insert(decisions).values(origins.map(origin => ({ companyId,
originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, title: "Decision", body: "Choose",
options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }))).returning();
const recoveryRows = await db.insert(issueRecoveryActions).values(origins.map(origin => ({ companyId,
sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture",
nextAction: "Choose", updatedAt: old() }))).returning();
const service = attentionRetentionService(db);
const pages = [];
for (const sourceKind of ["decision", "recovery_action"] as const) {
statements.length = 0;
const page = await service.load({ companyId, sourceKind });
expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false);
expect(statements.length).toBeLessThanOrEqual(3);
pages.push(page.rows!);
}
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[99].id));
const hiddenDecision = pages[0].find(entry => entry.sourceId === decisionRows[99].id)!;
const hiddenRecovery = pages[1].find(entry => entry.sourceId === recoveryRows[99].id)!;
expect(await service.check(hiddenDecision)).toBeNull(); expect(await service.check(hiddenRecovery)).toBeNull();
expect(await state("decision", hiddenDecision.sourceId)).toBeNull(); expect(await state("recovery_action", hiddenRecovery.sourceId)).toBeNull();
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
});
it("batches 100 failed runs and rechecks late visibility and supersession before archive", async () => {
const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
title: randomUUID(), status: "todo", updatedAt: old() }))).returning();
const runs = await db.insert(heartbeatRuns).values(origins.map(origin => ({ companyId, agentId,
status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }))).returning();
await db.insert(heartbeatRunEvents).values(runs.map(run => ({ companyId, agentId, runId: run.id,
seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: batch fixture" })));
const service = attentionRetentionService(db), scope = { companyId, sourceKind: "failed_run" as const };
statements.length = 0;
const page = await service.load(scope);
expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false);
expect(statements.length).toBeLessThanOrEqual(6);
const hiddenRun = runs[98], supersededRun = runs[99];
const hidden = page.rows!.find(entry => entry.sourceId === hiddenRun.id)!;
const superseded = page.rows!.find(entry => entry.sourceId === supersededRun.id)!;
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[98].id));
await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded",
contextSnapshot: { issueId: origins[99].id }, createdAt: new Date(old().getTime() + 60_000), finishedAt: old() });
expect(await service.check(hidden)).toBeNull(); expect(await service.check(superseded)).toBeNull();
expect(await state("failed_run", hiddenRun.id)).toBeNull(); expect(await state("failed_run", supersededRun.id)).toBeNull();
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
expect((await service.load(scope)).complete).toBe(true);
const refreshed = await service.load(scope);
expect(refreshed.rows).toHaveLength(98);
expect(refreshed.rows!.some(entry => [hiddenRun.id, supersededRun.id].includes(entry.sourceId))).toBe(false);
});
it("batches 100 blocker graphs with exact terminal timestamps and fresh terminal visibility", async () => {
const roots = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
title: randomUUID(), status: "blocked", updatedAt: old(), blockedTransitionAt: old(),
unblockDescriptor: { owner: "board", action: "Pick an owner" } }))).returning();
const terminals = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
title: randomUUID(), status: "backlog", assigneeAgentId: agentId, updatedAt: old() }))).returning();
await db.insert(issueRelations).values(roots.map((root, index) => ({ companyId,
issueId: terminals[index].id, relatedIssueId: root.id, type: "blocks" })));
await db.execute(sql`update issues set updated_at = updated_at + interval '0.000321 seconds' where id = ${terminals[99].id}::uuid`);
const service = attentionRetentionService(db), scope = { companyId, sourceKind: "blocker_attention" as const };
statements.length = 0;
const page = await service.load(scope);
const queries = [...statements];
expect(page.rows).toHaveLength(200); expect(page.complete).toBe(false);
const retentionReads = queries.filter(query => query.includes('from "decision_retention"'));
expect(retentionReads).toHaveLength(200);
expect(queries.length - retentionReads.length).toBeLessThanOrEqual(30);
const terminal = page.rows!.find(entry => entry.rootId === roots[99].id && entry.variant === "terminal")!;
expect(terminal.sourceId).toBe(terminals[99].id);
const [expected] = await db.execute(sql`select updated_at::text as at,
ceil(extract(epoch from updated_at) * 1000 + ${delay}) as due from issues where id = ${terminals[99].id}::uuid`);
expect(terminal.activityAt).toBe(expected.at); expect(terminal.dueAt).toBe(Number(expected.due));
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, terminals[99].id));
expect(await service.check(terminal)).toBeNull();
expect(await state("blocker_attention", terminals[99].id)).toBeNull();
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
expect((await service.load(scope)).complete).toBe(true);
const refreshed = await service.load(scope);
expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false);
});
it.each(["depth", "nodes"] as const)("preserves shallow terminal selection when an unrelated graph hits its %s limit", async limit => {
const root = await insertIssue("blocked", { updatedAt: new Date() });
const middle = await insertIssue("todo", { updatedAt: new Date() });
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId });
const heavy = await insertIssue("blocked", { updatedAt: new Date() });
const count = limit === "depth" ? BLOCKER_ATTENTION_MAX_DEPTH + 1 : BLOCKER_ATTENTION_MAX_NODES;
const heavyNodes = await db.insert(issues).values(Array.from({ length: count }, () => ({ companyId,
title: randomUUID(), status: "todo", updatedAt: new Date() }))).returning();
await db.insert(issueRelations).values([
{ companyId, issueId: middle.id, relatedIssueId: root.id, type: "blocks" },
{ companyId, issueId: terminal.id, relatedIssueId: middle.id, type: "blocks" },
...heavyNodes.map((node, index) => ({ companyId, issueId: node.id,
relatedIssueId: limit === "nodes" || index === 0 ? heavy.id : heavyNodes[index - 1].id, type: "blocks" })),
]);
const owner = issueService(db);
const expected = (await owner.listBlockerAttention(companyId, [root])).get(root.id)!;
const limited = (await owner.listBlockerAttention(companyId, [heavy])).get(heavy.id)!;
expect(expected.terminalBlockerIssueId).toBe(terminal.id);
expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId);
for (const input of [[root, heavy], [heavy, root]]) {
const batch = await owner.listBlockerAttentionBatch(companyId, input);
if (!batch.deferredRootIds.has(root.id)) expect(batch.attention.get(root.id)).toEqual(expected);
if (!batch.deferredRootIds.has(heavy.id)) expect(batch.attention.get(heavy.id)).toEqual(limited);
expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
if (limit === "depth") expect(batch.deferredRootIds.has(root.id)).toBe(false);
}
const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" });
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
expect(entry.dueAt).toBeLessThanOrEqual(Date.now());
if (limit === "depth") expect(entry.sourceId).toBe(terminal.id);
await attentionRetentionService(db).check(entry);
expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date);
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1);
});
it("keeps root-specific reachability and cycle budgets when another root explores farther", async () => {
const root = await insertIssue("blocked");
const chain = await db.insert(issues).values(Array.from({ length: BLOCKER_ATTENTION_MAX_DEPTH + 1 }, (_, index) => ({
companyId, title: randomUUID(), status: index === BLOCKER_ATTENTION_MAX_DEPTH ? "in_progress"
: index === BLOCKER_ATTENTION_MAX_DEPTH - 2 ? "blocked" : "todo", updatedAt: old() }))).returning();
const overlapping = chain[BLOCKER_ATTENTION_MAX_DEPTH - 2];
const cycle = await insertIssue("blocked");
const leaf = await insertIssue("backlog", { assigneeAgentId: agentId });
await db.insert(issueRelations).values([
...chain.map((node, index) => ({ companyId, issueId: node.id, relatedIssueId: index ? chain[index - 1].id : root.id, type: "blocks" })),
{ companyId, issueId: leaf.id, relatedIssueId: cycle.id, type: "blocks" },
{ companyId, issueId: cycle.id, relatedIssueId: leaf.id, type: "blocks" },
]);
const owner = issueService(db), roots = [root, overlapping, cycle];
const singles = new Map();
for (const item of roots) singles.set(item.id, (await owner.listBlockerAttention(companyId, [item])).get(item.id));
expect(singles.get(root.id).blockingTreeLive).toBe(false);
expect(singles.get(overlapping.id).blockingTreeLive).toBe(true);
for (const input of [roots, [...roots].reverse()]) {
const batch = await owner.listBlockerAttention(companyId, input);
for (const item of roots) expect(batch.get(item.id)).toEqual(singles.get(item.id));
}
});
it("bounds every graph query when independent root trees contain over 65535 unfinished nodes", async () => {
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at)
select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz
from generate_series(1, 70) n`);
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, updated_at)
select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-branch-' || n, 'todo', ${old().toISOString()}::timestamptz
from issues root cross join generate_series(1, 10) n
where root.company_id = ${companyId}::uuid and root.status = 'blocked'`);
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
select gen_random_uuid(), ${companyId}::uuid, branch.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
from issues branch cross join generate_series(1, 99) n
where branch.company_id = ${companyId}::uuid and branch.title like 'parameter-branch-%'`);
const completed = await insertIssue("blocked");
const shallow = await insertIssue("blocked", { updatedAt: new Date() });
const middle = await insertIssue("todo", { updatedAt: new Date() });
const overdue = await insertIssue("backlog", { assigneeAgentId: agentId });
await db.insert(issueRelations).values([
{ companyId, issueId: middle.id, relatedIssueId: shallow.id, type: "blocks" },
{ companyId, issueId: overdue.id, relatedIssueId: middle.id, type: "blocks" },
]);
const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked")));
parameterCounts.length = 0;
const result = await issueService(db).listBlockerAttentionBatch(companyId, roots).catch(error => {
throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`);
});
expect(result.attention.size).toBe(72);
expect(result.deferredRootIds.has(completed.id)).toBe(false);
expect(result.attention.get(completed.id)?.terminalBlockerIssueId).toBe(completed.id);
expect([...result.attention.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true);
expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100);
expect(parameterCounts.length).toBeLessThanOrEqual(100);
expect(result.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
expect(result.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2);
expect(result.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6);
expect(result.deferredRootIds.has(shallow.id)).toBe(true);
const service = attentionRetentionService(db);
const page = await service.load({ companyId, sourceKind: "blocker_attention" });
const immediate = page.rows!.find(entry => entry.rootId === shallow.id && entry.variant === "terminal")!;
expect(immediate.dueAt).toBeLessThanOrEqual(Date.now());
await service.check(immediate);
expect((await state("blocker_attention", overdue.id))?.archivedAt).toBeInstanceOf(Date);
}, 120_000);
it("preserves observed live work when a single large root exhausts shared fetch capacity", async () => {
const root = await insertIssue("blocked");
const live = await insertIssue("in_progress", { assigneeAgentId: agentId });
await db.insert(issueRelations).values({ companyId, issueId: live.id, relatedIssueId: root.id, type: "blocks" });
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
select gen_random_uuid(), ${companyId}::uuid, ${root.id}::uuid, 'bounded-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
from generate_series(1, 3000) n`);
const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]);
expect(batch.deferredRootIds.has(root.id)).toBe(true);
expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true);
const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" });
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
expect(await service.check(entry)).toBeNull();
expect(await state("blocker_attention", root.id)).toBeNull();
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
});
it.each(["in_progress", "run", "confirmation"])("preserves observed child live evidence (%s) after explicit blockers fill the node budget", async (kind) => {
const root = await insertIssue("blocked");
await db.execute(sql`with blockers as (
insert into issues (id, company_id, title, status, assignee_agent_id, updated_at)
select gen_random_uuid(), ${companyId}::uuid, 'explicit-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
from generate_series(1, 1999) n returning id)
insert into issue_relations (company_id, issue_id, related_issue_id, type)
select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`);
const child = await insertIssue(kind === "in_progress" ? "in_progress" : "todo", { parentId: root.id, assigneeAgentId: agentId });
if (kind === "run") {
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, child.id));
} else if (kind === "confirmation") {
await db.insert(issueThreadInteractions).values({ companyId, issueId: child.id, kind: "request_confirmation", status: "pending", payload: { prompt: "Approve?" } });
}
parameterCounts.length = 0;
const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]);
expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2);
expect(batch.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6);
expect(parameterCounts.length).toBeLessThanOrEqual(100);
expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100);
expect(batch.deferredRootIds.has(root.id)).toBe(true);
expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true);
const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" });
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
expect(await service.check(entry)).toBeNull();
expect(await db.select().from(decisionRetention)).toHaveLength(0);
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
});
it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => {
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at)
select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz
from generate_series(1, 4) n`);
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
select gen_random_uuid(), ${companyId}::uuid, root.id, 'held-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
from issues root cross join generate_series(1, 600) n
where root.company_id = ${companyId}::uuid and root.status = 'blocked'`);
await db.execute(sql`insert into decision_retention (company_id, source_kind, source_id, source_activity_at, keep)
select company_id, 'blocker_attention', id::text, updated_at, true from issues
where company_id = ${companyId}::uuid and status = 'backlog'`);
const service = attentionRetentionService(db), check = vi.spyOn(service, "check");
await start(service); await quiet();
expect(check).toHaveBeenCalledTimes(4);
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
});
it("bounds high-fanin ordinary issue writes to one coalescible hint", async () => {
const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load");
await start(service); await settle(); load.mockClear();
const leaf = await insertIssue("in_review", { updatedAt: new Date(), assigneeUserId: "board" });
await db.execute(sql`with parents as (
insert into issues (id, company_id, title, status)
select gen_random_uuid(), ${companyId}::uuid, 'fan-in-' || n, 'todo'
from generate_series(1, 2100) n returning id)
insert into issue_relations (company_id, issue_id, related_issue_id, type)
select ${companyId}::uuid, ${leaf.id}::uuid, id, 'blocks' from parents`);
const hints: string[] = [];
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
if (event === "settled") hints.push(...values ?? []);
});
let release!: () => void, entered = false;
const gate = new Promise<void>(resolve => { release = resolve; });
load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; });
statements.length = 0;
try {
await issueService(db).update(leaf.id, { title: "High fan-in ordinary edit 0" });
await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 });
for (let n = 1; n < 8; n++) await issueService(db).update(leaf.id, { title: `High fan-in ordinary edit ${n}` });
} finally { release(); unsubscribe(); }
expect(statements.filter(query => query.includes("with recursive affected"))).toHaveLength(8);
expect(hints).toHaveLength(8);
expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId })]));
await settle();
expect(load.mock.calls.length).toBeLessThanOrEqual(3);
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true);
let query!: Parameters<typeof db.execute>[0], returned = 0;
await db.transaction(async tx => {
const execute = tx.execute.bind(tx);
vi.spyOn(tx, "execute").mockImplementation(async statement => {
const result = await execute(statement);
if (new PgDialect().sqlToQuery(statement as SQL).sql.includes("with recursive affected")) {
query = statement; returned = result.length;
}
return result;
});
await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: leaf.id });
});
expect(returned).toBe(101);
const plans = await db.execute(sql`explain (analyze, format json) ${query as SQL}`);
const plan = (plans[0]["QUERY PLAN"] as any)[0].Plan;
expect(plan["Node Type"]).toBe("Limit");
expect(plan["Actual Rows"]).toBe(101);
const nodes: any[] = [], collect = (node: any) => { nodes.push(node); for (const child of node.Plans ?? []) collect(child); };
collect(plan);
const recursive = nodes.filter(node => node["Node Type"] === "Recursive Union");
const relations = nodes.filter(node => node["Relation Name"] === "issue_relations");
expect(recursive).toHaveLength(1);
expect(recursive[0]["Actual Rows"]).toBe(101);
expect(relations.length).toBeGreaterThan(0);
expect(relations.every(node => node["Actual Rows"] * node["Actual Loops"] <= 101)).toBe(true);
process.stdout.write(JSON.stringify({ reverseFanInPlan: plan }) + "\n");
});
it("retains far ancestor deadlines when a cyclic chain exceeds the precise hint budget", async () => {
const service = attentionRetentionService(db), load = vi.spyOn(service, "load");
await start(service); await settle(); load.mockClear();
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at, assignee_user_id)
select gen_random_uuid(), ${companyId}::uuid, 'chain-' || n,
case when n = 150 then 'in_review' else 'todo' end,
case when n = 150 then ${new Date(Date.now() - delay + 15_000).toISOString()}::timestamptz else now() end,
case when n = 150 then 'board' else null end
from generate_series(0, 150) n`);
await db.execute(sql`update issues child set parent_id = parent.id from issues parent
where child.company_id = ${companyId}::uuid and parent.company_id = child.company_id
and parent.title = 'chain-' || ((substring(child.title from 7)::integer + 1) % 151)`);
const [leaf, far] = await Promise.all([db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-0"))).then(rows => rows[0]),
db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-150"))).then(rows => rows[0])]);
await issueService(db).update(leaf.id, { title: "Ordinary deep chain edit" });
await settle();
expect(load.mock.calls.some(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true);
expect(await state("review", far.id)).toBeNull();
expect(coordinator!.nextWakeAt()).not.toBeNull();
await archived("review", far.id);
await quiet();
});
it("canonicalizes a complete cyclic affected set into one bounded batch hint", async () => {
const a = await insertIssue(), b = await insertIssue(), c = await insertIssue();
await db.update(issues).set({ parentId: b.id }).where(eq(issues.id, a.id));
await db.update(issues).set({ parentId: c.id }).where(eq(issues.id, b.id));
await db.update(issues).set({ parentId: a.id }).where(eq(issues.id, c.id));
const hints: string[] = [];
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
if (event === "settled") hints.push(...values ?? []);
});
try {
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: a.id }));
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: c.id }));
const wide = { companyId, issueIds: [b.id, a.id, c.id, a.id], ignored: {} as unknown };
wide.ignored = wide;
await db.transaction(tx => notifyAttentionRetentionWork(tx, wide));
} finally { unsubscribe(); }
expect(hints).toHaveLength(3);
expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId, issueIds: [a.id, b.id, c.id].sort() })]));
});
it("canonicalizes wide circular notification variables to the supported identity fields", async () => {
const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = [];
const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown };
wide.ignored = wide;
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
if (event === "settled") hints.push(...values ?? []);
});
try { await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); } finally { unsubscribe(); }
expect(hints).toEqual([JSON.stringify({ companyId, sourceKind: "review", sourceId, issueId })]);
});
it("keeps held, burst and sustained ordinary issue writes scoped while an unrelated deadline fires", async () => {
const row = await issueService(db).create(companyId, { title: "Active review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId });
const urgent = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date(Date.now() - delay + 10_000) });
const otherCompany = randomUUID();
await db.insert(companies).values({ id: otherCompany, name: "Unrelated", issuePrefix: "BURST" });
const foreign = await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany,
title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))).returning();
await db.insert(decisionRetention).values(foreign.map(issue => ({ companyId: otherCompany, sourceKind: "review", sourceId: issue.id, sourceActivityAt: issue.updatedAt, keep: true })));
const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load");
await start(service); await settle(); load.mockClear();
let release!: () => void, entered = false;
const gate = new Promise<void>(resolve => { release = resolve; });
load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; });
statements.length = 0;
await issueService(db).update(row.id, { title: "Held mutation" });
await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 });
await Promise.all(Array.from({ length: 5 }, (_, index) => issueService(db).update(row.id, { title: `Burst ${index}` })));
release(); await settle();
for (let index = 0; index < 8; index++) {
await issueService(db).update(row.id, { title: `Steady ${index}` }); await vi.advanceTimersByTimeAsync(1000); await settle();
}
await archived("review", urgent.id);
const routine = load.mock.calls.map(([scope]) => scope).filter(scope => !scope?.sourceId);
expect(routine.length).toBeGreaterThan(0);
expect(routine.every(scope => scope?.companyId === companyId && scope.issueId === row.id)).toBe(true);
const scans = statements.filter(query => query.includes("with source_candidates"));
expect(scans.length).toBeGreaterThan(0);
expect(scans.every(query => !query.includes("budget_incidents") && !query.includes("agent_error_alert"))).toBe(true);
expect(await state("review", row.id)).toBeNull();
});
it("wakes all linked issue sources when an ordinary visibility mutation reveals their origin", async () => {
const origin = await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board", responsibleUserId: "board" });
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }).returning();
await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: visibility" });
const [decision] = await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id,
title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }).returning();
const [recovery] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }).returning();
const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }).returning();
await start(); await quiet();
await issueService(db).update(origin.id, { hiddenAt: null });
for (const [kind, id] of [["decision", decision.id], ["recovery_action", recovery.id], ["failed_run", run.id], ["issue_thread_interaction", interaction.id]]) await archived(kind, id);
await settle(); expect(await state("review", origin.id)).toBeNull(); expect(coordinator!.nextWakeAt()).not.toBeNull();
});
it("invalidates old and new parents, terminal visibility and repointed blocker relations", async () => {
const oldParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" });
const newParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" });
const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" });
const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear();
await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle();
const ids = load.mock.calls.flatMap(([scope]) => scope?.issueIds ?? (scope?.issueId ? [scope.issueId] : []));
expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id]));
load.mockClear();
await issueService(db).update(terminal.id, { hiddenAt: null }); await settle();
expect(await state("blocker_attention", terminal.id)).toBeNull();
const visible = await service.load({ companyId, issueId: newParent.id });
expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true);
expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id || scope?.issueIds?.includes(newParent.id))).toBe(true);
const next = await insertIssue("backlog", { assigneeAgentId: agentId });
await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id);
});
it("cancels and reactivates a blocker deadline through ordinary liveness incident creation and hiding", async () => {
const root = await insertIssue("blocked");
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(Date.now() - delay + 10_000) });
await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" });
await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
const incident = await issueService(db).create(companyId, { title: "Resolve liveness", status: "todo", assigneeUserId: "board",
createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
originId: buildIssueGraphLivenessIncidentKey({ companyId, issueId: root.id, state: "needs_attention", blockerIssueId: terminal.id }) });
await quiet(); expect(await state("blocker_attention", terminal.id)).toBeNull();
await issueService(db).update(incident.id, { hiddenAt: new Date() });
await archived("blocker_attention", terminal.id); await quiet();
});
it("resolves opaque run issue references through trusted company issue rows", async () => {
const issue = await insertIssue();
const [valid, opaque] = await db.insert(heartbeatRuns).values([
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: issue.id } },
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: "none" } },
]).returning();
const hints: string[] = [];
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
if (event === "settled") hints.push(...values ?? []);
});
try {
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: valid.id }));
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: opaque.id }));
} finally { unsubscribe(); }
expect(hints).toEqual([JSON.stringify({ companyId, issueId: issue.id }), JSON.stringify({ companyId })]);
});
it("accepts ordinary liveness incident creation and completion with a none leaf sentinel", async () => {
const source = await insertIssue("blocked", { responsibleUserId: "board" });
const service = issueService(db);
const incident = await service.create(companyId, { title: "Invalid participant", status: "todo", assigneeUserId: "board",
createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
originId: `harness_liveness:${companyId}:${source.id}:invalid_review_participant:none` });
await service.update(source.id, { blockedByIssueIds: [incident.id] });
await service.update(incident.id, { status: "done" });
expect((await service.getById(incident.id))?.status).toBe("done");
expect((await service.getRelationSummaries(source.id)).blockedBy).toEqual([]);
});
it.each(["hidden", "done"] as const)("reactivates an older review when moved stranded recovery is %s", async transition => {
const review = await insertIssue("in_review", { assigneeAgentId: agentId, responsibleUserId: "board" });
const parent = await insertIssue("todo", { responsibleUserId: "board" });
const recovery = await insertIssue("todo", { parentId: review.id, originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,
originId: review.id, assigneeUserId: "board", responsibleUserId: "board" });
await issueService(db).update(recovery.id, { parentId: parent.id });
expect((await issueService(db).listReviewAttention(companyId, [review])).get(review.id)?.state).not.toBe("stalled");
await start(); await quiet();
await issueService(db).update(recovery.id, transition === "hidden" ? { hiddenAt: new Date() } : { status: "done" });
await settle();
expect((await state("review", review.id))?.archivedAt).toBeInstanceOf(Date);
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, review.id))).toHaveLength(1);
});
it("scopes ordinary execution-control writes to their company", async () => {
const otherCompany = randomUUID();
await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" });
await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany,
title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() })));
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
const service = attentionRetentionService(db), load = vi.spyOn(service, "load");
await start(service); await quiet(); load.mockClear();
for (let count = 0; count < 3; count++) {
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization,
tx => tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)), { companyId });
await quiet();
}
expect(load).toHaveBeenCalled();
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true);
});
it("registers and cancels budget deadlines through ordinary policy and incident producers", async () => {
const service = budgetService(db);
await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 100,
warnPercent: 85, hardStopEnabled: false, notifyEnabled: true }, "board");
const retention = attentionRetentionService(db), load = vi.spyOn(retention, "load");
await start(retention); await quiet(); load.mockClear();
const [event] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai",
billingType: "metered_api", model: "fixture", inputTokens: 100, outputTokens: 20,
costCents: 90, occurredAt: new Date() }).returning();
await service.evaluateCostEvent(event);
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
const [incident] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId));
expect(incident).toMatchObject({ status: "open", thresholdType: "soft" });
expect(await state("budget_alert", incident.id)).toBeNull();
await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 200 }, "board");
await quiet();
expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0].status).toBe("resolved");
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true);
});
it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => {
const origin = await insertIssue();
const review = await insertIssue("in_review", { assigneeUserId: "board" });
const blocker = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Choose a dependency" }, blockedTransitionAt: old() });
await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board" });
const hidden = await insertIssue("todo", { hiddenAt: old() });
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), updatedAt: old(), finishedAt: old() }).returning();
await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: fixture" });
await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id,
title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() });
await db.insert(issueThreadInteractions).values([
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() },
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeAgentId: agentId, updatedAt: old() },
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeUserId: "board", updatedAt: old() },
{ companyId, issueId: hidden.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() },
]);
await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() });
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId, windowKind: "calendar_month_utc", amount: 100 }).returning();
await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId,
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: old(), windowEnd: new Date(), thresholdType: "hard", amountLimit: 100, amountObserved: 100, updatedAt: old() });
await db.insert(agents).values({ companyId, name: "Error", role: "engineer", status: "error", updatedAt: old() });
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true, includeDismissed: true, excludeSources: ["approval", "join_request"] });
const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort();
const actual = [...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort();
expect(new Set(feed.items.map(item => item.sourceKind)).size).toBe(8); expect(actual).toEqual(expected);
expect(actual).toContain(`review:${review.id}`); expect(actual).toContain(`blocker_attention:${blocker.id}`);
await start(); for (const item of feed.items) await archived(item.sourceKind, item.subject.id); await quiet();
});
it("keeps unseen sources, cancels their deadline, and reactivates when unkept or revived", async () => {
const row = await insertIssue("in_review", { assigneeUserId: "board" });
await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: true });
await start(); await quiet(); expect((await state("review", row.id))?.archivedAt).toBeNull();
await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: false });
await archived("review", row.id); await quiet();
await decisionRetentionService(db).revive(mutation("review", row.id)); await archived("review", row.id); await quiet();
});
it("moves deadlines on source activity and cancels when the source leaves attention", async () => {
const row = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date() });
await start(); await settle(); const first = coordinator!.nextWakeAt()!;
await change(tx => tx.update(issues).set({ updatedAt: new Date(Date.now() - delay + 600_000) }).where(eq(issues.id, row.id)));
await settle(); expect(coordinator!.nextWakeAt()).toBeLessThan(first);
await issueService(db).update(row.id, { status: "todo" }); await quiet(); expect(await state("review", row.id)).toBeNull();
});
it("preserves outer transaction ownership and first writes during drain", async () => {
await start(); await quiet(); draining = true;
let id: string | undefined;
await db.transaction(async outer => {
await outer.transaction(async nested => {
await notifyAttentionRetentionWork(nested, { companyId });
id = (await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db)).id;
});
await vi.advanceTimersByTimeAsync(60_000); expect(await state("review", id!)).toBeNull();
});
await vi.advanceTimersByTimeAsync(120_000); expect(await state("review", id!)).toBeNull();
draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", id!); await quiet();
await expect(db.transaction(async outer => {
await outer.transaction(async nested => { await notifyAttentionRetentionWork(nested, { companyId }); await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db); });
throw new Error("rollback");
})).rejects.toThrow("rollback");
await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(1);
});
it("isolates companies, catches up at activation, and bounds retained-history pages", async () => {
const other = randomUUID(); await db.insert(companies).values({ id: other, name: "Paused", issuePrefix: "RP", status: "paused" });
const row = await insertIssue("in_review", { companyId: other, assigneeUserId: "board" });
await db.insert(issues).values(Array.from({ length: 1000 }, () => ({ companyId, title: randomUUID(), status: "done", updatedAt: old() })));
const service = attentionRetentionService(db); const page = await service.load({ companyId });
expect(page).toEqual({ rows: [], complete: true });
await start(service); await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(0);
await companyService(db).update(other, { status: "active" });
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(1); });
expect((await db.select().from(decisionRetention))[0]).toMatchObject({ companyId: other, sourceId: row.id }); await quiet();
});
it("preserves terminal blocker source identity and its own activity deadline", async () => {
const root = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Pick an owner" }, blockedTransitionAt: old() });
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(old().getTime() + 60_000) });
await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" });
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
const entries = await loadAll();
const expected = feed.items.filter(item => item.sourceKind === "blocker_attention")
.map(item => `${item.subject.id}:${new Date(item.activityAt).toISOString()}`).sort();
expect(entries.filter(item => item.sourceKind === "blocker_attention")
.map(item => `${item.sourceId}:${new Date(item.activityAt).toISOString()}`).sort()).toEqual(expected);
expect(expected).toContain(`${terminal.id}:${terminal.updatedAt.toISOString()}`);
await start(); await archived("blocker_attention", terminal.id); await archived("blocker_attention", root.id); await quiet();
expect((await state("blocker_attention", terminal.id))?.sourceActivityAt).toEqual(terminal.updatedAt);
expect((await state("blocker_attention", root.id))?.sourceActivityAt).toEqual(root.blockedTransitionAt);
});
it("matches newest confirmations, independent gates, org invokability, and conversation visibility", async () => {
const origin = await insertIssue();
const conversation = await insertIssue("in_review", { conversationAgentId: agentId, conversationUserId: "board", conversationState: "waiting", assigneeAgentId: agentId });
const hidden = await insertIssue("todo", { hiddenAt: old() });
const [paused, manager, unhealthy] = await db.insert(agents).values([
{ companyId, name: "Paused", role: "engineer", status: "paused" },
{ companyId, name: "Terminated", role: "manager", status: "terminated" },
{ companyId, name: "Unhealthy", role: "engineer", status: "active" },
]).returning();
await db.update(agents).set({ reportsTo: manager.id }).where(eq(agents.id, unhealthy.id));
const common = { companyId, issueId: origin.id, kind: "request_confirmation", payload: { prompt: "Approve?" }, createdAt: old(), updatedAt: old() };
const [older, newest, independent] = await db.insert(issueThreadInteractions).values([
common, { ...common, createdAt: new Date(old().getTime() + 1000) },
{ ...common, payload: { prompt: "Independent", qualityReview: { artifactKey: "copy" } } },
{ ...common, createdAt: new Date(), addresseeUserId: "board" },
{ ...common, addresseeAgentId: agentId },
{ ...common, issueId: hidden.id },
{ ...common, issueId: conversation.id, kind: "ask_user_questions", payload: { questions: [] } },
{ ...common, issueId: conversation.id, addresseeAgentId: paused.id },
{ ...common, issueId: conversation.id, addresseeAgentId: unhealthy.id, kind: "ask_user_questions", payload: { questions: [] } },
] as typeof issueThreadInteractions.$inferInsert[]).returning();
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort();
expect([...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort()).toEqual(expected);
expect(expected).not.toContain(`issue_thread_interaction:${older.id}`);
expect(expected).toContain(`issue_thread_interaction:${newest.id}`); expect(expected).toContain(`issue_thread_interaction:${independent.id}`);
await start(); await archived("issue_thread_interaction", newest.id); await archived("issue_thread_interaction", independent.id); await quiet();
await change(tx => tx.update(issueThreadInteractions).set({ status: "cancelled" }).where(eq(issueThreadInteractions.id, newest.id)));
await archived("issue_thread_interaction", older.id); await quiet();
});
it("excludes maintained reviews until a pending approval creates human attention", async () => {
const row = await insertIssue("in_review", { assigneeAgentId: agentId });
const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: row.id,
kind: "request_confirmation", addresseeAgentId: agentId, payload: { prompt: "Agent verdict" } }).returning();
expect((await loadAll()).some(entry => entry.sourceKind === "review")).toBe(false);
const [approval] = await db.insert(approvals).values({ companyId, type: "hire_agent", payload: {}, status: "pending" }).returning();
await db.insert(issueApprovals).values({ companyId, issueId: row.id, approvalId: approval.id });
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
expect(feed.items.some(item => item.sourceKind === "review" && item.subject.id === row.id)).toBe(true);
await start(); await archived("review", row.id); await quiet();
expect(await state("issue_thread_interaction", interaction.id)).toBeNull();
});
it("reactivates an exhausted failed run through the ordinary durable event producer", async () => {
const origin = await insertIssue();
const [run, newer] = await db.insert(heartbeatRuns).values([
{ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() },
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: origin.id }, createdAt: new Date(old().getTime() + 1000), finishedAt: old() },
]).returning();
await start(); await quiet();
await appendHeartbeatRunEvent(db, { companyId, agentId, runId: run.id, eventType: "lifecycle", message: "Bounded retry exhausted after 3 attempts" });
await quiet(); expect(await state("failed_run", run.id)).toBeNull();
await change(tx => tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, newer.id)));
await archived("failed_run", run.id); await quiet();
});
it("cancels an agent error deadline through ordinary clearError", async () => {
await db.update(agents).set({ status: "error", updatedAt: new Date() }).where(eq(agents.id, agentId));
await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
await createAgentLifecycle(db).clearError(agentId); await quiet();
expect(await state("agent_error_alert", agentId)).toBeNull();
});
it("drains a paged backlog with source-scoped self notifications and leaves completed history quiet", async () => {
await db.insert(issues).values(Array.from({ length: 225 }, () => ({ companyId, title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() })));
const service = attentionRetentionService(db);
const load = vi.spyOn(service, "load");
await start(service);
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(225); }, { timeout: 30_000 });
await quiet();
const scopes = load.mock.calls.map(([scope]) => scope);
expect(scopes.filter(scope => !scope)).toHaveLength(3);
expect(scopes.filter(scope => scope && !scope.sourceId)).toHaveLength(0);
expect(scopes.filter(scope => scope?.sourceId).length).toBeLessThanOrEqual(225);
expect(new Set((await db.select().from(decisionRetention)).map(row => row.sourceId)).size).toBe(225);
});
it("retries a failed atomic archive across drain and preserves notifications during a scan", async () => {
const row = await insertIssue("in_review", { assigneeUserId: "board" });
await db.execute(sql`create function attention_retention_fail() returns trigger language plpgsql as $$ begin raise exception 'retention injected'; end $$`);
await db.execute(sql`create trigger attention_retention_fail before insert on decision_retention for each row execute function attention_retention_fail()`);
await start(); expect(errors).toHaveLength(1); errors.length = 0;
expect(await state("review", row.id)).toBeNull(); expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
draining = true; await db.execute(sql`drop trigger attention_retention_fail on decision_retention`);
statements.length = 0; await vi.advanceTimersByTimeAsync(120_000);
expect(statements.length).toBeLessThanOrEqual(1);
expect(statements.every(query => query.includes("pg_xact_status"))).toBe(true);
statements.length = 0; await vi.advanceTimersByTimeAsync(300_000); expect(statements).toEqual([]);
draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", row.id); await quiet();
await coordinator!.stop(); coordinator = undefined;
let release!: () => void; let scanned!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
const entered = new Promise<void>(resolve => { scanned = resolve; });
const service = attentionRetentionService(db), original = service.load;
vi.spyOn(service, "load").mockImplementationOnce(async (...args) => { const result = await original(...args); scanned(); await gate; return result; });
const starting = start(service); await entered;
const next = await change(async tx => insertIssue("in_review", { assigneeUserId: "board" }, tx));
release(); await starting; await archived("review", next.id); await quiet();
});
it("registers and cancels a deadline through ordinary interaction creation and withdrawal", async () => {
const origin = await insertIssue(); await start(); await quiet();
const service = issueThreadInteractionService(db);
const interaction = await service.create(origin, { kind: "request_confirmation", payload: { version: 1, prompt: "Approve the result?" } }, { userId: "board" });
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(await state("issue_thread_interaction", interaction.id)).toBeNull();
await service.withdrawInteraction(origin, interaction.id, { reason: "No longer needed" }, { userId: "board" }); await quiet();
});
});
@@ -62,7 +62,7 @@ describeEmbeddedPostgres("attention service", () => {
beforeAll(async () => {
if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-attention-service-");
db = createDb(externalUrl ?? tempDb!.connectionString);
db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 });
}, 30_000);
afterEach(async () => {
@@ -0,0 +1,109 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, projects, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { budgetService } from "../services/budgets.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
vi.mock("../services/activity-log.js", () => ({ logActivity: vi.fn() }));
const support = await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("budget history for deleted scopes", () => {
let db: ReturnType<typeof createDb>;
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase("paperclip-budget-history-");
db = createDb(database.connectionString);
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
afterEach(async () => {
await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents);
await db.delete(projects); await db.delete(agents); await db.delete(companies);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
async function fixture() {
const companyId = randomUUID(), agentId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "History company", issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Current agent", role: "engineer" });
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId,
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 300 }).returning();
await db.insert(costEvents).values({ companyId, agentId, provider: "openai", model: "history-fixture", inputTokens: 1, outputTokens: 1, costCents: 17, occurredAt: new Date() });
return { companyId, agentId, policy: policy! };
}
it.each(["agent", "project"] as const)("keeps current budgets and deleted %s policy/incident history readable without changing stored rows", async scopeType => {
const { companyId, policy } = await fixture(), deletedId = randomUUID();
if (scopeType === "agent") await db.insert(agents).values({ id: deletedId, companyId, name: "Former agent", role: "engineer" });
else await db.insert(projects).values({ id: deletedId, companyId, name: "Former project" });
const [historical] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: deletedId,
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 200 }).returning();
await db.insert(budgetIncidents).values({ companyId, policyId: historical!.id, scopeType, scopeId: deletedId,
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
thresholdType: "hard", amountLimit: 200, amountObserved: 201 });
if (scopeType === "agent") await db.delete(agents).where(eq(agents.id, deletedId));
else await db.delete(projects).where(eq(projects.id, deletedId));
const policiesBefore = await db.select().from(budgetPolicies), incidentsBefore = await db.select().from(budgetIncidents), costsBefore = await db.select().from(costEvents);
const overview = await budgetService(db).overview(companyId);
expect(overview.policies).toHaveLength(2);
expect(overview.policies.find(row => row.policyId === policy.id)).toMatchObject({ companyId, scopeName: "Current agent", amount: 300, observedAmount: 17 });
expect(overview.policies.find(row => row.policyId === historical!.id)).toMatchObject({ companyId, scopeType, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, amount: 200 });
expect(overview.activeIncidents).toMatchObject([{ companyId, policyId: historical!.id, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, status: "open", amountObserved: 201 }]);
expect(await db.select().from(budgetPolicies)).toEqual(policiesBefore);
expect(await db.select().from(budgetIncidents)).toEqual(incidentsBefore);
expect(await db.select().from(costEvents)).toEqual(costsBefore);
});
it("rejects missing-scope budget grants but can dismiss their historical incidents", async () => {
const { companyId, agentId } = await fixture(), missingId = randomUUID(), service = budgetService(db);
await expect(service.evaluateAdmission(companyId, missingId)).rejects.toThrow("Agent not found");
await expect(service.evaluateAdmission(companyId, agentId, { projectId: missingId })).rejects.toThrow("Project not found");
for (const scopeType of ["agent", "project"] as const) {
await expect(service.upsertPolicy(companyId, { scopeType, scopeId: missingId, amount: 100 }, "board")).rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: missingId,
windowKind: "calendar_month_utc", amount: 100 }).returning();
const [approval] = await db.insert(approvals).values({ companyId, type: "budget_override_required", payload: {} }).returning();
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, approvalId: approval!.id, scopeType, scopeId: missingId,
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
thresholdType: "hard", amountLimit: 100, amountObserved: 101 }).returning();
const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) };
await expect(service.resolveIncident(companyId, incident!.id, { action: "raise_budget_and_resume", amount: 200 }, "board"))
.rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }).toEqual(before);
const dismissed = await service.resolveIncident(companyId, incident!.id, { action: "keep_paused" }, "board");
expect(dismissed).toMatchObject({ companyId, policyId: policy!.id, scopeId: missingId, scopeName: `Deleted ${scopeType}`, status: "dismissed" });
const { status: _status, resolvedAt: _resolvedAt, updatedAt: _updatedAt, ...preserved } = incident!;
const [saved] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident!.id));
expect(saved).toMatchObject({ ...preserved, status: "dismissed", resolvedAt: expect.any(Date) });
expect((await db.select().from(approvals).where(eq(approvals.id, approval!.id)))[0]).toMatchObject({ status: "rejected" });
expect(await db.select().from(budgetPolicies)).toEqual(before.policies);
expect(await db.select().from(agents)).toEqual(before.agents);
expect(await db.select().from(costEvents)).toEqual(before.costs);
expect((await service.overview(companyId)).activeIncidents.some(row => row.id === incident!.id)).toBe(false);
}
});
it("rejects a corrupted policy pointing to another company's scope instead of exposing its name", async () => {
const { companyId } = await fixture(), foreignCompanyId = randomUUID(), foreignAgentId = randomUUID();
await db.insert(companies).values({ id: foreignCompanyId, name: "Other company", issuePrefix: `F${foreignCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
await db.insert(agents).values({ id: foreignAgentId, companyId: foreignCompanyId, name: "Private other-company agent", role: "engineer" });
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: foreignAgentId, windowKind: "calendar_month_utc", amount: 200 }).returning();
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, scopeType: "agent", scopeId: foreignAgentId,
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
thresholdType: "hard", amountLimit: 200, amountObserved: 201 }).returning();
const service = budgetService(db);
await expect(service.overview(companyId)).rejects.toThrow("Budget scope not found");
for (const scopeId of [foreignAgentId, randomUUID()]) {
await db.update(budgetIncidents).set({ scopeId }).where(eq(budgetIncidents.id, incident!.id));
const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
agents: await db.select().from(agents), costs: await db.select().from(costEvents) };
for (const action of ["raise_budget_and_resume", "keep_paused"] as const) {
await expect(service.resolveIncident(companyId, incident!.id, { action, amount: 300 }, "board")).rejects.toThrow("Budget scope not found");
expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
agents: await db.select().from(agents), costs: await db.select().from(costEvents) }).toEqual(before);
}
}
});
});
@@ -16341,9 +16341,15 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
await tx.update(toolConnections).set({ credentialSecretRefs: connection.credentialSecretRefs.map(ref => ({ ...ref,
versionSelector: ref.versionSelector === "1" ? "latest" : "1" })) }).where(eq(toolConnections.id, connection.id));
});
await expect.poll(async () => ((await f.checkpoint()).value as Record<string, unknown>).credentialFingerprint,
{ timeout: 5000 }).not.toBe(oldWindow.credentialFingerprint);
expect((await f.checkpoint()).value).toMatchObject({ generation: oldWindow.generation, outcome: "epoch_initialized" });
// Credential synchronization and epoch initialization commit separately.
// Require the final epoch receipt, not just the intermediate fingerprint.
await expect.poll(async () => {
const value = (await f.checkpoint()).value as Record<string, unknown>;
return { ...value, fingerprintChanged: typeof value.credentialFingerprint === "string"
&& value.credentialFingerprint !== oldWindow.credentialFingerprint };
}, { timeout: 5000 }).toMatchObject({
fingerprintChanged: true, generation: oldWindow.generation, outcome: "epoch_initialized",
});
expect(f.requests).toEqual([]);
} finally { await coordinator.stop(); }
});
@@ -4168,6 +4168,7 @@ describe("company portability", () => {
const insertedRelationValues: Array<Record<string, unknown>> = [];
const monitorUpdates: Array<Record<string, unknown>> = [];
const db = {
transaction: async (write: (tx: unknown) => Promise<unknown>) => write(db),
insert: () => ({
values: (rows: Array<Record<string, unknown>>) => ({
onConflictDoNothing: async () => {
@@ -4184,6 +4185,7 @@ describe("company portability", () => {
}),
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as any;
db.transaction = async (write: (tx: typeof db) => Promise<unknown>) => write(db);
return { db, insertedRelationValues, monitorUpdates };
}
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { randomUUID } from "node:crypto";
import { sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
@@ -28,12 +29,13 @@ import {
import { issueReadSqlCondition, projectReadSqlCondition } from "../services/authorization.js";
import { issueService } from "../services/issues.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_COMPANY_SEARCH_TEST_DATABASE_URL");
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres company search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
`Skipping embedded Postgres company search tests on this host: ${("reason" in embeddedPostgresSupport ? embeddedPostgresSupport.reason : "unsupported environment")}`,
);
}
@@ -80,8 +82,11 @@ describeEmbeddedPostgres("companySearchService", () => {
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-");
db = createDb(tempDb.connectionString);
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-");
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
// External suites can reuse this dedicated database sequentially. Match
// the fresh embedded fixture before this suite creates its own rows.
if (externalDatabaseUrl) await db.execute(sql`TRUNCATE TABLE companies CASCADE`);
svc = companySearchService(db);
await db.execute(sql.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm"));
}, 20_000);
@@ -100,6 +105,7 @@ describeEmbeddedPostgres("companySearchService", () => {
});
afterAll(async () => {
await db?.$client.end({ timeout: 1 });
await tempDb?.cleanup();
});
+131 -1
View File
@@ -1,4 +1,5 @@
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { installDatabaseWorkSignals, signalDatabaseWork, databaseWorkPending } from "../../../packages/db/src/work-signals.js";
import { afterEach, expect, it, vi } from "vitest";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { registerDeadlineWork } from "../services/deadline-work.js";
@@ -39,3 +40,132 @@ it.each([
expect(run).toHaveBeenCalledTimes(2);
expect(onError).not.toHaveBeenCalled();
});
it("releases a settled first-write hold during drain and retains a probe-free resume wake", async () => {
vi.useFakeTimers();
let draining = false;
const owner = installDatabaseWorkSignals({
execute: vi.fn(async () => [{ status: "committed" }]),
transaction: async (callback: (tx: object) => Promise<unknown>) => callback({ execute: async () => [{ xid: "1" }] }),
});
const run = vi.fn(async () => {}), next = vi.fn(async () => null);
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() });
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 });
await worker.ready;
draining = true;
const before = idleWorkSnapshot().active;
await owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); });
await vi.advanceTimersByTimeAsync(0);
expect(idleWorkSnapshot().active).toBe(before);
expect(coordinator.nextWakeAt()).not.toBeNull();
await vi.advanceTimersByTimeAsync(3 * 30_000);
expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled();
draining = false;
await vi.advanceTimersByTimeAsync(30_000);
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
});
it("retains failed work through drain and preserves a future domain deadline", async () => {
vi.useFakeTimers();
let draining = false;
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
const run = vi.fn(async () => { if (run.mock.calls.length === 1) { draining = true; throw new Error("retry debt"); } });
const deadline = Date.now() + 600_000;
const next = vi.fn(async () => deadline);
const onError = vi.fn();
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError });
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 });
await worker.ready;
await vi.advanceTimersByTimeAsync(90_000);
expect(run).toHaveBeenCalledOnce(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled();
draining = false; await vi.advanceTimersByTimeAsync(30_000);
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBe(deadline);
draining = true; await vi.advanceTimersByTimeAsync(30_000);
expect(coordinator.nextWakeAt()).toBe(deadline); expect(onError).toHaveBeenCalledOnce();
});
it("does no domain probes in warm standby and reconstructs after admission opens", async () => {
vi.useFakeTimers();
let standby = true;
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
const run = vi.fn(async () => {}), next = vi.fn(async () => null);
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !standby, canReconcile: () => !standby, onError: vi.fn() });
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !standby, retryMs: 30_000 });
await worker.ready; await vi.advanceTimersByTimeAsync(90_000);
expect(run).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled();
standby = false; await vi.advanceTimersByTimeAsync(30_000);
expect(run).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull();
});
it("does not release an ambiguous database intent while dispatch is drained", async () => {
vi.useFakeTimers();
let draining = false, committed = false;
const owner = installDatabaseWorkSignals({
execute: vi.fn(async () => [{ status: committed ? "committed" : "in progress" }]),
transaction: async (callback: (tx: object) => Promise<unknown>) => {
await callback({ execute: async () => [{ xid: "2" }] });
throw new Error("lost acknowledgement");
},
});
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() });
const run = vi.fn(async () => {});
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next: async () => null, canRun: () => !draining, retryMs: 30_000 });
await worker.ready; draining = true;
const before = idleWorkSnapshot().active;
await expect(owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); })).rejects.toThrow("lost acknowledgement");
await vi.advanceTimersByTimeAsync(0);
expect(idleWorkSnapshot().active).toBe(before + 1);
expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(true);
committed = true; await vi.advanceTimersByTimeAsync(30_000);
expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(false);
expect(idleWorkSnapshot().active).toBe(before);
draining = false; await vi.advanceTimersByTimeAsync(30_000);
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
});
it("backs off an expired deadline when only the domain is drained and resumes promptly", async () => {
vi.useFakeTimers();
let paused = false;
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
const run = vi.fn(async () => {});
const domainCanRun = vi.fn(() => !paused);
const deadline = Date.now() + 1_000;
const next = vi.fn(async () => run.mock.calls.length === 1 ? deadline : null);
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() });
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, {
run, next, canRun: domainCanRun, retryMs: 30_000,
});
await worker.ready;
paused = true;
const before = idleWorkSnapshot();
await vi.advanceTimersByTimeAsync(1_000);
expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000);
const attempts = domainCanRun.mock.calls.length;
await vi.advanceTimersByTimeAsync(90_000);
expect(domainCanRun.mock.calls.length - attempts).toBeLessThanOrEqual(12);
// Four attempts (expiry plus three retries), each begins and ends tracked work.
expect(idleWorkSnapshot().generation - before.generation).toBe(8);
expect(idleWorkSnapshot().active).toBe(before.active);
expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled();
paused = false;
await vi.advanceTimersByTimeAsync(30_000);
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
});
it("retains a prompt paused wake when a new request preempts a far-future deadline", async () => {
vi.useFakeTimers();
let paused = false;
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
const run = vi.fn(async () => {});
const deadline = Date.now() + 86_400_000;
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() });
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, {
run, next: async () => run.mock.calls.length === 1 ? deadline : null, canRun: () => !paused, retryMs: 30_000,
});
await worker.ready; paused = true; worker.wake();
await vi.advanceTimersByTimeAsync(0);
expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000);
expect(owner.execute).not.toHaveBeenCalled();
paused = false; await vi.advanceTimersByTimeAsync(30_000);
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
});
@@ -73,7 +73,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes
const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({
status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId,
updatedAt, inputValues: {}, metadata: { continuationPending: true },
}).where(eq(decisions.id, id)));
}).where(eq(decisions.id, id)), { companyId });
async function start() {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
onError: error => errors.push(error) });
@@ -467,7 +467,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes
} else {
await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running",
chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000),
metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)));
metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)), { companyId });
}
await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet();
const [retained] = await db.select().from(decisionRetention);
@@ -793,7 +793,7 @@ describePg("decisionService", () => {
expect((await service().get(id))?.metadata.continuationPending).toBe(false);
}
const markReady = (id: string, source = db) => writeDecisionChange(source, async tx => tx.update(decisions)
.set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)));
.set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)), { companyId });
it("coalesces sequential tenant writes without dropping pending recovery", async () => {
const row = await createCommentDecision();
@@ -0,0 +1,427 @@
import { setImmediate as immediate } from "node:timers/promises";
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
import { accessService } from "../services/access.js";
import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js";
import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js";
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
import { issueTreeControlService } from "../services/issue-tree-control.js";
import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyNativeStatusWakeWork, notifyTaskRetrySettlement } from "../services/deferred-execution-notifications.js";
import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-receipt-work.js";
import { heartbeatService } from "../services/heartbeat.js";
import { requestTaskRetry } from "../services/task-retry.js";
import { issueService } from "../services/issues.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
vi.setConfig({ testTimeout: 90_000 });
describe("deferred receipt durable dispatch", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
let heartbeat: ReturnType<typeof heartbeatService>;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
let drain = false;
const errors: unknown[] = [];
const passes = new Set<Promise<unknown>>();
beforeAll(async () => {
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL");
if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-");
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
});
beforeEach(async () => {
await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0;
heartbeat = heartbeatService(db);
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
onError: error => errors.push(error) });
const register = coordinator.register.bind(coordinator);
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
const pass = Promise.resolve(task.run(signal)); passes.add(pass);
void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass;
} });
});
afterEach(async () => {
await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
async function fixture() {
const companyId = randomUUID(), actorId = "deferred-receipt-user";
await db.insert(authUsers).values({ id: actorId, name: "Receipt operator", email: `${actorId}@example.test`,
createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
await db.insert(companies).values({ id: companyId, name: "Deferred dispatch", issuePrefix: `DE${companyId.slice(0, 6)}`,
defaultResponsibleUserId: actorId, requireBoardApprovalForNewAgents: false });
await db.insert(companyMemberships).values({ companyId, principalId: actorId, principalType: "user", status: "active", membershipRole: "owner" });
const [agent] = await db.insert(agents).values({ companyId, name: "Receipt process", adapterType: "process", status: "idle",
adapterConfig: { command: process.execPath, args: ["-e", "console.log('receipt-proof')"], cwd: process.cwd() },
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning();
const [issue] = await db.insert(issues).values({ companyId, title: "Resume saved task", status: "todo", assigneeAgentId: agent.id }).returning();
const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, issueId: issue.id,
status: "failed", finishedAt: new Date(), responsibleUserId: actorId, errorCode: "process_failed",
contextSnapshot: { issueId: issue.id }, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
return { companyId, agent, issue, source, actorId };
}
async function quiet() {
await vi.waitFor(async () => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute");
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
await vi.advanceTimersByTimeAsync(20 * 60_000);
expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed);
}
async function finish(issueId: string) {
await issueService(db).update(issueId, { status: "done" });
await vi.waitFor(async () => {
await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions();
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
}, { timeout: 15_000 });
const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"));
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
if (saved.length) {
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
while (Date.now() <= due) {
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
await Promise.allSettled([...passes]); await immediate();
}
}
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(500); await Promise.allSettled([...passes]);
await heartbeat.drainActiveRunExecutions();
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0);
});
}
it("is quiet after empty reconstruction and terminal receipt history", async () => {
const f = await fixture();
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } });
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet();
});
it("dispatches an ordinary committed task Retry into saved process output", async () => {
const f = await fixture();
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
observedRunId: f.source.id, idempotencyKey: randomUUID() });
expect(receipt.accepted).toBe(true);
let runId: string | undefined;
await vi.waitFor(async () => {
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!));
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId!;
}, { timeout: 15_000 });
await vi.waitFor(async () => {
await heartbeat.drainActiveRunExecutions();
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
expect(run.stdoutExcerpt).toContain("receipt-proof");
}, { timeout: 15_000 });
await finish(f.issue.id);
const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`));
expect(linked).toHaveLength(2);
expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId]));
expect(linked.map(row => row.idempotencyKey)).toContain(`task-retry-dispatch:${receipt.requestId}`);
expect(linked.find(row => row.id === receipt.requestId)?.status).toBe("coalesced");
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1);
await quiet();
});
it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => {
const f = await fixture(), updatedAt = new Date(), native = kind === "native claim";
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
source: "automation", status: native ? "claimed" : "deferred_issue_execution", claimedAt: updatedAt, updatedAt,
payload: native ? {} : { executionWait: { reason: "recovery" } },
requestedByActorType: "system", requestedByActorId: native ? "native-status-committer" : null }).returning();
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
const check = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, scope.requestId!));
});
await registerDeferredReceiptWork(db, coordinator, { resume: check, settleTaskChat: vi.fn(), dispatchNative: check,
schedulingSuppressed: async () => false }, () => !drain).ready;
const due = updatedAt.getTime() + (native ? 60_000 : 30_000);
expect(coordinator.nextWakeAt()).toBe(due); expect(check).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(check).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
await vi.waitFor(() => expect(check).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id })));
await quiet();
});
it("reconciles the first drained producer and waits for resume without domain SQL", async () => {
const f = await fixture(); drain = true;
const resume = vi.fn(async () => {}), dispatchNative = vi.fn(async () => {});
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative,
schedulingSuppressed: async () => false }, () => !drain).ready;
await db.transaction(async tx => {
await notifyDeferredExecutionWork(tx, { companyId: f.companyId });
await tx.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, source: "on_demand",
status: "deferred_issue_execution", reason: "task_retry" });
});
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
await vi.advanceTimersByTimeAsync(1000);
expect(resume).not.toHaveBeenCalled(); expect(dispatchNative).not.toHaveBeenCalled();
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
drain = false; await vi.advanceTimersByTimeAsync(30_000);
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(1)); await quiet();
});
it.each([
{ name: "saved comment", payload: { _paperclipWakeContext: { wakeCommentIds: ["saved-comment"] } } },
{ name: "interaction", payload: { mutation: "interaction" } },
{ name: "execution wait", payload: { executionWait: { reason: "recovery" } } },
{ name: "interrupt", payload: { queuedCommentInterrupt: { actorId: "operator" } } },
])("activates only the committed company's $name receipt after an empty start", async ({ payload }) => {
const f = await fixture(), foreign = await fixture();
const resume = vi.fn(async () => {});
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
schedulingSuppressed: async () => false }, () => !drain).ready;
const [other] = await db.insert(agentWakeupRequests).values({ companyId: foreign.companyId, agentId: foreign.agent.id,
source: "on_demand", status: "deferred_issue_execution", payload }).returning();
let requestId: string;
await db.transaction(async tx => {
await tx.transaction(async nested => {
await notifyDeferredExecutionWork(nested, { companyId: f.companyId });
const [row] = await nested.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: new Date(Date.now() - 31_000) }).returning();
requestId = row.id;
});
expect(resume).not.toHaveBeenCalled();
});
await vi.waitFor(() => expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId })), { timeout: 5_000 });
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, other.id)))[0].status).toBe("deferred_issue_execution");
await quiet();
});
it("reconstructs a cancelled legacy run's exact deferred queue even without comment payload", async () => {
const f = await fixture();
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
source: "on_demand", status: "deferred_issue_execution" }).returning();
await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { queuedCommentInterruptQueueId: wake.id } })
.where(eq(heartbeatRuns.id, f.source.id));
const resume = vi.fn(async () => {});
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
schedulingSuppressed: async () => false }, () => !drain).ready;
expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id }));
await quiet();
});
it("settles a processed chat receipt without restaging it until an actual eligibility commit", async () => {
const f = await fixture();
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Receipt chat", type: "mcp_http" }).returning();
const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Receipt chat", transport: "rest_api" }).returning();
const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection.id,
provider: "slack", publicId: randomUUID(), assignedAgentId: f.agent.id }).returning();
const intentId = randomUUID(), actionId = randomUUID(), previous = new Date(Date.now() - 60_000);
await db.insert(agentWakeupRequests).values([
{ id: intentId, companyId: f.companyId, agentId: f.agent.id, source: "on_demand", reason: "task_retry",
status: "deferred_issue_execution", updatedAt: previous, payload: { issueId: f.issue.id,
taskRetry: { schema: "paperclip.task-retry.v1", generationId: intentId, sourceRunId: f.source.id,
actorId: f.actorId, strategy: "evaluate", chatActionId: actionId } } },
{ id: actionId, companyId: f.companyId, agentId: f.agent.id, source: "automation", status: "deferred_issue_execution",
payload: { issueId: f.issue.id, taskRetryRequestId: intentId } },
]);
await db.transaction(async tx => {
await notifyTaskRetrySettlement(tx, { companyId: f.companyId, requestId: intentId });
await tx.insert(chatActions).values({ id: actionId, companyId: f.companyId, endpointId: endpoint.id,
providerActionId: randomUUID(), kind: "failed_run_retry", status: "processed", updatedAt: new Date(),
payload: { issueId: f.issue.id, taskRetryIntentId: intentId, failedRunId: f.source.id } });
});
await vi.waitFor(async () => {
const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, intentId));
expect(intent.updatedAt.getTime()).toBeGreaterThan(previous.getTime());
}, { timeout: 5_000 });
expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status).toBe("processed");
await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
await vi.waitFor(async () => expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status)
.toBe("issued"), { timeout: 5_000 });
await quiet();
});
it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => {
const f = await fixture(), initialAt = new Date();
const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({
companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId,
}))).returning();
const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId },
_paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } };
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: initialAt }).returning();
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps);
await registerDeferredReceiptWork(db, coordinator, { resume: scope => queue.resumeDeferredExecutionReceipts(scope),
settleTaskChat: async () => {}, dispatchNative: async () => {}, schedulingSuppressed: async () => false }, () => !drain).ready;
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
const movedAt = new Date();
const writer = createQueuedCommentIssueLockWriter(db, { syncCommentReferences: async () => {},
deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {} });
await writer.withLockedQueue({ issue: f.issue, queueId: wake.id,
actor: { actorType: "user", actorId: f.actorId, agentId: null, runId: null, agentApiKeyId: null } },
async (_locked, tx) => { await tx.updateWakeQueuedCommentIds({ wakeId: wake.id, payload,
ids: comments.map(row => row.id).reverse(), updatedAt: movedAt }); });
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
expect(deps.resumeQueuedCommentInterrupt).not.toHaveBeenCalled();
expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
await vi.waitFor(() => expect(deps.resumeQueuedCommentInterrupt).toHaveBeenCalledExactlyOnceWith(f.companyId, wake.id));
await quiet();
});
it("reactivates a held task Retry when the authorized subtree hold is released without wakeAgents", async () => {
const f = await fixture(), tree = issueTreeControlService(db);
const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
const hold = await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor });
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
observedRunId: f.source.id, idempotencyKey: randomUUID() });
expect(request.requestId).toBeTruthy();
await vi.waitFor(async () => {
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
expect(wake.payload?.executionWait).toMatchObject({ reason: "issue_tree_hold_active" });
expect(wake.runId).toBeNull();
}, { timeout: 5_000 });
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
await tree.releaseHold(f.companyId, f.issue.id, hold.hold.id, { actor, reason: "Resume saved Retry" });
let runId: string | null = null;
await vi.waitFor(async () => {
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
}, { timeout: 40_000 });
await vi.waitFor(async () => {
await heartbeat.drainActiveRunExecutions();
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
expect(run.stdoutExcerpt).toContain("receipt-proof");
}, { timeout: 15_000 });
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
await finish(f.issue.id); await quiet();
});
it("keeps a moved native dispatch error deadline instead of retrying at its old time", async () => {
const f = await fixture(), initialAt = new Date();
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
source: "automation", status: "queued", error: "Transient dispatch failure", updatedAt: initialAt,
requestedByActorType: "system", requestedByActorId: "native-status-committer" }).returning();
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
const dispatchNative = vi.fn(async () => {});
await registerDeferredReceiptWork(db, coordinator, { resume: async () => {}, settleTaskChat: async () => {},
dispatchNative, schedulingSuppressed: async () => false }, () => !drain).ready;
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
const movedAt = new Date();
await db.update(agentWakeupRequests).set({ updatedAt: movedAt }).where(eq(agentWakeupRequests.id, wake.id));
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
expect(dispatchNative).not.toHaveBeenCalled(); expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
await vi.waitFor(() => expect(dispatchNative).toHaveBeenCalledTimes(1)); await quiet();
});
it("arms one new wait reason grace and stays quiet when that same gate is still closed", async () => {
const f = await fixture(); let reason = "operator_pause";
const resume = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
await writeDeferredExecutionWait(db, { companyId: scope.companyId!, requestId: scope.requestId! }, { reason });
});
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {},
schedulingSuppressed: async () => false }, () => !drain).ready;
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
source: "on_demand", status: "deferred_issue_execution", reason: "task_retry" }).returning();
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
for (const nextReason of ["operator_pause", "budget_pause"]) {
reason = nextReason;
await writeDeferredExecutionWait(db, { companyId: f.companyId, requestId: wake.id }, { reason });
const [current] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
const due = current.updatedAt.getTime() + 30_000;
await vi.advanceTimersByTimeAsync(1_000);
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due));
const checked = resume.mock.calls.length;
await vi.advanceTimersByTimeAsync(due - Date.now());
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(checked + 1));
await quiet();
}
expect(resume).toHaveBeenCalledTimes(2);
});
it("reactivates a saved Retry through ordinary restore cancelled task", async () => {
const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
observedRunId: f.source.id, idempotencyKey: randomUUID() });
const tree = issueTreeControlService(db);
const cancel = await tree.createHold(f.companyId, f.issue.id, { mode: "cancel", actor });
await tree.cancelIssueStatusesForHold(f.companyId, f.issue.id, cancel.hold.id);
const restore = await tree.createHold(f.companyId, f.issue.id, { mode: "restore", actor });
const commitEligibility = () => tree.restoreIssueStatusesForHold(f.companyId, f.issue.id, restore.hold.id, { actor });
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 });
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull();
await commitEligibility();
let runId: string | null = null;
await vi.waitFor(async () => {
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
}, { timeout: 15_000 });
await vi.waitFor(async () => {
await heartbeat.drainActiveRunExecutions();
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
expect(run.stdoutExcerpt).toContain("receipt-proof");
}, { timeout: 15_000 });
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
await finish(f.issue.id); await quiet();
});
it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => {
const f = await fixture(), departing = `departing-${randomUUID()}`;
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
observedRunId: f.source.id, idempotencyKey: randomUUID() });
await db.insert(authUsers).values({ id: departing, name: "Departing member", email: `${departing}@example.test`,
createdAt: new Date(), updatedAt: new Date() });
const [member] = await db.insert(companyMemberships).values({ companyId: f.companyId, principalId: departing,
principalType: "user", status: "active", membershipRole: "member" }).returning();
await issueService(db).update(f.issue.id, { assigneeUserId: departing, assigneeAgentId: null });
const readIntent = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0];
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
const deferred = vi.fn(), native = vi.fn();
const unsubscribes = [subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred),
subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents, native)];
try {
await expect(db.transaction(async tx => {
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
{ reassignment: { assigneeAgentId: f.agent.id } });
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
throw new Error("roll back member archive");
})).rejects.toThrow("roll back member archive");
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].assigneeUserId).toBe(departing);
await reconcileDatabaseWork(db, DELIVERY_QUEUES.deferredExecutionReceipts);
await reconcileDatabaseWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents);
deferred.mockClear(); native.mockClear();
await db.transaction(async tx => {
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
{ reassignment: { assigneeAgentId: f.agent.id } });
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
});
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({
assigneeUserId: null, assigneeAgentId: f.agent.id,
});
expect((await db.select().from(companyMemberships).where(eq(companyMemberships.id, member.id)))[0].status).toBe("archived");
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0);
} finally { for (const unsubscribe of unsubscribes) unsubscribe(); }
await quiet();
});
it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => {
const f = await fixture(); const dispatchNative = vi.fn(async () => {});
await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative,
schedulingSuppressed: async () => false }, () => !drain).ready;
await expect(db.transaction(async tx => {
await notifyNativeStatusWakeWork(tx, { companyId: f.companyId });
throw new Error("outer rollback");
})).rejects.toThrow("outer rollback");
await db.transaction(async tx => {
await expect(tx.transaction(async nested => {
await notifyNativeStatusWakeWork(nested, { companyId: f.companyId }); throw new Error("nested rollback");
})).rejects.toThrow("nested rollback");
});
await quiet(); expect(dispatchNative).not.toHaveBeenCalled();
});
});
@@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => {
const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000);
for (const at of [due, earlier]) {
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns)
.set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)));
.set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId });
await settle(); expect(c.nextWakeAt()).toBe(at.getTime());
}
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns)
.set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)));
.set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId });
await quiet(c);
});
it("delivers a reconciled continuation when marking and resolving commit together", async () => {
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { randomUUID } from "node:crypto";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -29,9 +30,9 @@ import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { createGitHubExternalObjectProvider } from "../services/github-external-object-provider.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECTS_TEST_DATABASE_URL");
const describeEmbeddedPostgres = externalDatabaseUrl || embeddedPostgresSupport.supported ? describe : describe.skip;
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
if (!embeddedPostgresSupport.supported) {
console.warn(
@@ -73,7 +73,7 @@ describeEmbeddedPostgres("git restore point retention", () => {
} = {}) {
const companyId = input.companyId ?? randomUUID();
if (!input.companyId) {
await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.slice(0, 6)}` });
await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.replaceAll("-", "")}` });
}
const issueStatus = input.issueStatus ?? "done";
const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS);
@@ -19,11 +19,20 @@ import { secretService } from "../services/secrets.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { aiConnectionService } from "../services/ai-connections.js";
import * as aiRuntime from "../services/ai-connection-runtime.js";
import * as connectorRuntime from "../services/connector-runtime.js";
import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js";
import * as sentry from "../sentry.js";
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>(complete => { resolve = complete; });
return { promise, resolve };
}
const skillFailures = vi.hoisted(() => new Map<string, string>());
const skillWaits = vi.hoisted(() => new Map<string, () => Promise<void>>());
describe("Runner home mode during recovery preparation", () => {
it.each([
@@ -52,6 +61,7 @@ vi.mock("../services/company-skills.js", async importOriginal => {
return { ...service, withRuntimeSkillEntries: async <T>(companyId: string,
options: Parameters<typeof service.withRuntimeSkillEntries>[1],
consume: (entries: Awaited<ReturnType<typeof service.listRuntimeSkillEntries>>) => T | Promise<T>) => {
await skillWaits.get(companyId)?.();
const failure = skillFailures.get(companyId);
if (failure) throw new Error(failure);
return service.withRuntimeSkillEntries(companyId, options, consume);
@@ -91,7 +101,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }),
});
}, 60_000);
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); });
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); skillWaits.clear(); });
afterAll(async () => {
await heartbeat?.drainActiveRunExecutions();
unregisterServerAdapter(adapterType);
@@ -288,6 +298,45 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } });
}, 60_000);
it("overlaps skills with assignments and dependent connection instructions before composition", async () => {
const f = await seed();
const skillGate = deferred<void>();
skillWaits.set(f.companyId, () => skillGate.promise);
const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments");
const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun");
const configured = await configuration(f, { provider: "codex", model: "gpt-6.1-sol", homeMode: "isolated" });
const opening = configured.prepare();
try {
await vi.waitFor(() => {
expect(assignments).toHaveBeenCalledOnce();
expect(instructions).toHaveBeenCalledOnce();
});
} finally {
skillGate.resolve();
}
expect(await opening).toMatchObject({ prepared: true, runtimeConfig: { homeMode: "isolated", model: "gpt-6.1-sol" } });
});
it("settles concurrent configuration reads before exposing the first preparation failure", async () => {
const f = await seed();
const assignmentGate = deferred<Awaited<ReturnType<typeof connectorRuntime.resolveConnectorAssignments>>>();
const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments").mockReturnValue(assignmentGate.promise);
const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun").mockRejectedValue(new Error("later instruction failure"));
skillFailures.set(f.companyId, "first skill failure");
const configured = await configuration(f, { provider: "codex", homeMode: "isolated" });
let settled = false;
const opening = configured.prepare();
void opening.then(() => { settled = true; }, () => { settled = true; });
try {
await vi.waitFor(() => expect(assignments).toHaveBeenCalledOnce());
await Promise.resolve();
expect(settled).toBe(false);
expect(instructions).not.toHaveBeenCalled();
} finally {
assignmentGate.resolve([]);
}
await expect(opening).rejects.toThrow("first skill failure");
});
it("does not install new environment declarations when recovering a saved skill selection", async () => {
const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`;
@@ -445,11 +494,11 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
const runIdHash = createHash("sha256").update(finished.id).digest("hex").slice(0, 12);
const steps = exporter.getFinishedSpans().filter(span => span.name.startsWith("heartbeat.prepare.configuration_")
&& span.attributes["paperclip.task.run.run_id"] === runIdHash);
expect(steps.map(span => span.name)).toEqual([
expect(steps.map(span => span.name).sort()).toEqual([
"configuration_mentions", "configuration_github_identity", "configuration_secrets", "configuration_ai_runtime",
"configuration_skills", "configuration_connector_assignments", "configuration_connector_skills",
"configuration_connector_delivery", "configuration_connection_instructions", "configuration_agent_identity",
].map(step => `heartbeat.prepare.${step}`));
].map(step => `heartbeat.prepare.${step}`).sort());
for (const span of steps) {
expect(span.status).toEqual({ code: 0 });
expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
@@ -494,7 +543,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
&& span.attributes["paperclip.task.run.run_id"] === runIdHash);
expect(steps.find(span => span.name === "heartbeat.prepare.configuration_ai_runtime")?.status.code).toBe(0);
expect(steps.find(span => span.name === "heartbeat.prepare.configuration_skills")?.status).toEqual({ code: 2 });
expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(false);
expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(true);
for (const span of steps) {
expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
expect(span.events).toEqual([]);
@@ -0,0 +1,127 @@
import { idleWorkSnapshot } from "../services/task-admission.js";
import { PgDialect } from "drizzle-orm/pg-core";
import { afterEach, describe, expect, it, vi } from "vitest";
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
import { registerRunDispatchWork } from "../services/heartbeat/dispatch-work.js";
const cleanup: Array<() => Promise<void>> = [];
afterEach(async () => {
for (const stop of cleanup.splice(0)) await stop();
vi.useRealTimers();
});
function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean } = {}) {
vi.useFakeTimers();
vi.setSystemTime(1_000_000);
const dialect = new PgDialect();
const execute = vi.fn(async (query: any) => {
if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
throw new Error("unexpected database query");
});
const db: { execute: typeof execute; transaction: (callback: (tx: any) => Promise<any>) => Promise<any> } = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise<any>): Promise<any> {
return callback({ execute, transaction: db.transaction });
} });
const state = { due: input.due ?? null as number | null, suppressed: input.suppressed ?? false,
draining: input.draining ?? false, standby: false, fail: false };
const promote = vi.fn(async () => {
if (state.fail) throw new Error("database unavailable");
if (state.due !== null && state.due <= Date.now()) state.due = null;
});
const next = vi.fn(async () => state.due);
const dispatch = vi.fn(async (_scope?: { companyId: string }) => {});
const coordinator = createDeliveryWorkCoordinator({ owner: db,
canRun: () => !state.draining && !state.standby, canReconcile: () => !state.standby, onError: vi.fn() });
const worker = registerRunDispatchWork(coordinator, {
promoteDueScheduledRetries: promote, nextScheduledRetryAt: next,
resumeQueuedRunsAfterDrain: dispatch, schedulingSuppressed: async () => state.suppressed,
}, () => !state.draining && !state.standby);
cleanup.push(() => coordinator.stop());
return { db, state, promote, next, dispatch, coordinator, worker,
notify: (scope?: { companyId: string }) => db.transaction(tx => notifyRunDispatch(tx, scope)),
settle: () => vi.advanceTimersByTimeAsync(1_000) };
}
describe("run dispatch obligations", () => {
it.each([false, true])("settles empty/feature-suppressed state and stops all owner work (suppressed=%s)", async suppressed => {
const f = fixture({ suppressed }); await f.worker.ready;
const calls = [f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length];
expect(f.coordinator.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(20 * 60_000);
expect([f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]).toEqual(calls);
expect(f.db.execute).not.toHaveBeenCalled();
});
it("restores an exact deadline and does no earlier recovery work", async () => {
const f = fixture({ due: 1_010_000 }); await f.worker.ready;
expect(f.coordinator.nextWakeAt()).toBe(1_010_000);
await vi.advanceTimersByTimeAsync(9_999); expect(f.promote).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(1); expect(f.promote).toHaveBeenCalledTimes(2);
expect(f.state.due).toBeNull(); expect(f.coordinator.nextWakeAt()).toBeNull();
expect(f.dispatch).toHaveBeenCalledOnce();
});
it("rearms earlier work and cancels a future deadline on a committed producer change", async () => {
const f = fixture({ due: 1_600_000 }); await f.worker.ready;
f.state.due = 1_005_000; await f.notify(); await f.settle();
expect(f.coordinator.nextWakeAt()).toBe(1_005_000);
f.state.due = null; await f.notify(); await f.settle();
expect(f.coordinator.nextWakeAt()).toBeNull();
const before = f.next.mock.calls.length;
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.next).toHaveBeenCalledTimes(before);
});
it("batches repeated tenant writes and passes only their company to queued dispatch", async () => {
const f = fixture(); await f.worker.ready;
for (let index = 0; index < 100; index++) {
const source = { companyId: "company-a", issueId: `issue-${index % 2}`,
runId: `run-${index}`, agentId: `agent-${index}`, interactionBody: "not a dispatch key" };
await f.notify(source);
await vi.advanceTimersByTimeAsync(10);
}
expect(f.promote.mock.calls.length).toBeLessThanOrEqual(2);
expect(f.dispatch.mock.calls.length).toBeLessThanOrEqual(2);
await f.settle();
expect(f.dispatch).toHaveBeenLastCalledWith({ companyId: "company-a" });
for (const [scope] of f.dispatch.mock.calls.slice(1)) expect(scope).toEqual({ companyId: "company-a" });
expect(f.coordinator.nextWakeAt()).toBeNull();
const count = f.dispatch.mock.calls.length;
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.dispatch).toHaveBeenCalledTimes(count);
});
it("remembers the first producer arriving during drain without owner SQL", async () => {
const f = fixture(); await f.worker.ready; f.state.draining = true;
await f.notify(); await f.settle(); await vi.advanceTimersByTimeAsync(90_000);
expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).toHaveBeenCalledOnce();
expect(idleWorkSnapshot().active).toBe(0);
f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
expect(f.promote).toHaveBeenCalledTimes(2); expect(f.dispatch).toHaveBeenCalledTimes(2);
expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("releases a failed producer hold during drain and retains recovery for resume", async () => {
const f = fixture(); await f.worker.ready;
f.state.fail = true; await f.notify(); await f.settle();
expect(f.promote).toHaveBeenCalledTimes(2);
f.state.draining = true; await vi.advanceTimersByTimeAsync(90_000);
expect(f.promote).toHaveBeenCalledTimes(2); expect(f.next).toHaveBeenCalledOnce();
expect(idleWorkSnapshot().active).toBe(0);
f.state.fail = false; f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("preserves a notification during a nonoverlapping pass", async () => {
const f = fixture(); await f.worker.ready;
let release!: () => void;
const held = new Promise<void>(resolve => { release = resolve; });
f.promote.mockImplementationOnce(() => held);
await f.notify(); await f.settle();
await f.notify(); await f.settle();
expect(f.promote).toHaveBeenCalledTimes(2);
release(); await f.settle();
await vi.advanceTimersByTimeAsync(30_000);
expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("does not dispatch on warm standby and cancels deadlines at shutdown", async () => {
const f = fixture({ due: 1_010_000 }); f.state.standby = true; await f.worker.ready;
await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).not.toHaveBeenCalled();
f.state.standby = false; await vi.advanceTimersByTimeAsync(30_000);
expect(f.promote).toHaveBeenCalledOnce(); expect(f.state.due).toBeNull();
f.state.due = Date.now() + 60_000; await f.notify(); await f.settle(); await f.worker.stop();
await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).toHaveBeenCalledTimes(2);
});
});
@@ -0,0 +1,238 @@
import { idleWorkSnapshot } from "../services/task-admission.js";
import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, companies, costEvents, createDb, databaseWorkPending, heartbeatRuns } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { agentService } from "../services/agents.js";
import { budgetService } from "../services/budgets.js";
import { heartbeatService } from "../services/heartbeat.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_DISPATCH_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("run dispatch production wiring (controlled local process adapter)", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
let heartbeat: ReturnType<typeof heartbeatService>;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
let worker: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]> | undefined;
let reads: ReturnType<typeof vi.spyOn>;
let draining = false;
const errors: unknown[] = [];
beforeAll(async () => {
if (!external) database = await startEmbeddedPostgresTestDatabase("run-dispatch-work-");
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
}, 90_000);
beforeEach(async () => {
await db.execute(sql`TRUNCATE companies CASCADE`);
heartbeat = heartbeatService(db); draining = false; errors.length = 0;
reads = vi.spyOn(db, "select");
});
afterEach(async () => {
await worker?.stop(); await coordinator?.stop();
await heartbeat.drainActiveRunExecutions();
worker = undefined; coordinator = undefined;
vi.useRealTimers(); vi.restoreAllMocks();
expect(errors).toEqual([]);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
async function start() {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining,
canReconcile: () => true, onError: error => errors.push(error) });
worker = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
await worker.ready;
}
async function fixture() {
const companyId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Dispatch", issuePrefix: `D${companyId.slice(0, 7)}`,
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "dispatch-user" });
const [agent] = await db.insert(agents).values({ companyId, name: "Controlled process", role: "engineer", status: "idle",
adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('dispatch-proof')"], cwd: process.cwd() },
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
}).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "failed",
responsibleUserId: "dispatch-user", errorCode: "codex_transient_upstream", finishedAt: new Date(),
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, contextSnapshot: {},
}).returning();
return { companyId, agentId: agent.id, runId: run.id };
}
async function quiet() {
await heartbeat.drainActiveRunExecutions();
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull(), { timeout: 5_000 });
const count = reads.mock.calls.length;
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
try {
await vi.advanceTimersByTimeAsync(20 * 60_000);
expect(reads).toHaveBeenCalledTimes(count);
} finally { vi.useRealTimers(); }
}
async function status(id: string) {
return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]);
}
async function completed(id: string) {
await vi.waitFor(async () => expect(["running", "succeeded"]).toContain((await status(id))?.status), { timeout: 10_000 });
await heartbeat.drainActiveRunExecutions();
const completed = await status(id);
expect(completed.status).toBe("succeeded");
expect(completed.stdoutExcerpt).toContain("dispatch-proof");
}
it("keeps fresh empty, configured idle and retained terminal history quiet", async () => {
await start(); await quiet();
await fixture(); await worker!.stop(); await coordinator!.stop();
await start(); await quiet();
});
it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => {
const f = await fixture(); await start(); await quiet();
const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 4_000, random: () => 0 });
expect(result.outcome).toBe("scheduled");
if (result.outcome !== "scheduled") throw new Error("Expected retry creation");
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime()), { timeout: 5_000 });
expect((await status(result.run.id)).status).toBe("scheduled_retry");
await new Promise(resolve => setTimeout(resolve, 100));
expect(Date.now()).toBeLessThan(result.dueAt.getTime());
expect((await status(result.run.id)).status).toBe("scheduled_retry");
await completed(result.run.id);
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(1);
await quiet();
});
it("reconstructs an overdue retry after process restart without a periodic tick", async () => {
const f = await fixture();
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 120_000), delayMs: 60_000 });
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
heartbeat = heartbeatService(db);
await start(); await completed(scheduled.run.id); await quiet();
});
it("cancels a future retry and removes its old deadline", async () => {
const f = await fixture(); await start();
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 });
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime()), { timeout: 5_000 });
await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true });
await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled");
});
it("retains the first retry written during drain and catches it up after resume", async () => {
const f = await fixture(); await start(); await quiet(); draining = true;
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 200 });
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
const count = reads.mock.calls.length;
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
try {
await vi.advanceTimersByTimeAsync(90_000);
expect(reads).toHaveBeenCalledTimes(count);
expect(idleWorkSnapshot().active).toBe(0);
} finally { vi.useRealTimers(); }
expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
draining = false; worker!.wake();
await completed(scheduled.run.id); await quiet();
});
it("uses a committed budget raise to resume a queued run in the company it paused", async () => {
const f = await fixture();
const budgets = budgetService(db);
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 100,
metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
const [cost] = await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agentId,
provider: "fixture", biller: "fixture", model: "controlled", inputTokens: 0, outputTokens: 0,
costCents: 125, occurredAt: new Date() }).returning();
await budgets.evaluateCostEvent(cost);
expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0].status).toBe("paused");
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
await start(); await quiet(); expect((await status(queued.id)).status).toBe("queued");
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 10_000,
metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
await completed(queued.id); await quiet();
});
it("clearing an agent error wakes saved queue work only after its own transaction commits", async () => {
const f = await fixture(); await start(); await quiet();
await db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId));
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
let writing: ReturnType<ReturnType<typeof agentService>["clearError"]> | undefined;
try {
await db.transaction(async blocker => {
await blocker.select().from(agents).where(eq(agents.id, f.agentId)).for("update");
writing = agentService(db).clearError(f.agentId);
void writing.catch(() => {});
await new Promise(resolve => setTimeout(resolve, 100));
expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(false);
expect((await status(queued.id)).status).toBe("queued");
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
});
} finally { await writing; }
await completed(queued.id); await quiet();
});
it("leaves cancellation-owned retries quiet even when their old promotion time is overdue", async () => {
const f = await fixture();
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 60_000), delayMs: 1_000 });
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
await db.update(heartbeatRuns).set({ resultJson: { startupCancellation: { requestedAt: new Date().toISOString() },
cancellation: { expected: true, reason: "operator stop", source: "operator" } } }).where(eq(heartbeatRuns.id, scheduled.run.id));
await start(); await quiet();
expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
expect(await heartbeat.promoteDueScheduledRetries()).toEqual({ promoted: 0, runIds: [] });
await quiet();
});
it("rejects an error clear without publishing work when the lifecycle is no longer ready", async () => {
const f = await fixture(); await start(); await quiet();
await db.update(agents).set({ status: "error", lifecycleState: "paused" }).where(eq(agents.id, f.agentId));
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
try {
await expect(agentService(db).clearError(f.agentId)).rejects.toThrow("Only agents in error status");
expect(coordinator!.nextWakeAt()).toBeNull();
const count = reads.mock.calls.length;
await vi.advanceTimersByTimeAsync(60_000);
expect(reads).toHaveBeenCalledTimes(count);
} finally { vi.useRealTimers(); }
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
await quiet();
});
it("keeps a foreign company's future retry intact while concurrent workers claim one due retry", async () => {
const selected = await fixture(), foreign = await fixture(); await start();
const later = await heartbeat.scheduleBoundedRetry(foreign.runId, { delayMs: 600_000 });
const due = await heartbeat.scheduleBoundedRetry(selected.runId, { delayMs: 500 });
if (later.outcome !== "scheduled" || due.outcome !== "scheduled") throw new Error("Expected retries");
const other = heartbeatService(db);
const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
const competingWorker = other.registerRunDispatchWork(competing, () => true);
try {
await competingWorker.ready; await completed(due.run.id); await other.drainActiveRunExecutions();
expect((await status(later.run.id)).status).toBe("scheduled_retry");
expect((await status(later.run.id)).scheduledRetryAt).toEqual(later.dueAt);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, selected.runId))).toHaveLength(1);
await heartbeat.cancelRun(later.run.id, "fixture teardown", { suppressImmediateRecovery: true });
await quiet();
} finally { await competingWorker.stop(); await competing.stop(); await other.drainActiveRunExecutions(); }
});
it("holds nested producer work until outer commit and rolls back without a successor", async () => {
const f = await fixture(); await start(); await quiet();
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
let queuedId = "";
const writing = db.transaction(async tx => {
await tx.transaction(async nested => {
await notifyRunDispatch(nested);
const [row] = await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
responsibleUserId: "dispatch-user", status: "queued", contextSnapshot: {} }).returning();
queuedId = row.id;
});
await held;
});
await vi.waitFor(() => expect(queuedId).not.toBe(""));
await new Promise(resolve => setTimeout(resolve, 100));
expect(await status(queuedId)).toBeUndefined();
release(); await writing; await completed(queuedId); await quiet();
await expect(db.transaction(async tx => {
await tx.transaction(async nested => { await notifyRunDispatch(nested); await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "queued" }); });
throw new Error("rollback outer");
})).rejects.toThrow("rollback outer");
await quiet();
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2);
});
});
@@ -262,6 +262,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
await heartbeat.resumeQueuedRuns();
const [delivered] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
expect(delivered.runId).toBeTruthy();
// The receipt can commit before execution enters liveRunExecutions.
// Observe actual adapter admission before awaiting its drain.
await vi.waitFor(async () => {
const current = await heartbeat.getRun(delivered.runId!);
expect(countExecuteCallsForRun(delivered.runId!), JSON.stringify({
status: current?.status, errorCode: current?.errorCode,
})).toBe(1);
}, { timeout: 5_000 });
await heartbeat.waitForRunExecutionDrain(delivered.runId!);
expect(countExecuteCallsForRun(delivered.runId!)).toBe(1);
expect((await heartbeat.getRun(delivered.runId!))?.contextSnapshot).toMatchObject({ wakeCommentIds: [comment.id] });
@@ -1,10 +1,11 @@
import { createHash, randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared";
import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js";
import { heartbeatService } from "../services/heartbeat.js";
import { prepareHeartbeatTaskContext } from "../services/heartbeat/task-context.js";
import { documentService } from "../services/documents.js";
import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js";
import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js";
@@ -19,6 +20,7 @@ if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${s
describePostgres("heartbeat task context delivered to an adapter", () => {
const adapterType = "task_context_test";
const captured = new Map<string, AdapterExecutionContext>();
const inspectPreparation = new Map<string, () => void>();
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let heartbeat: ReturnType<typeof heartbeatService>;
@@ -33,6 +35,7 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
type: adapterType,
supportsLocalAgentJwt: false,
execute: async (input) => {
inspectPreparation.get(input.agent.id)?.();
captured.set(input.runId, input);
return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} };
},
@@ -118,6 +121,101 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
});
});
it("reads selected comments once for both wake projections", async () => {
const f = await seed();
const commentId = randomUUID();
await db.insert(issueComments).values({
id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user",
body: "selected wake canary",
});
const select = vi.spyOn(db, "select");
inspectPreparation.set(f.agentId, () => {
const commentReads = select.mock.calls.filter(([projection]) => projection
&& projection.body === issueComments.body && projection.id === issueComments.id && projection.deletedAt === issueComments.deletedAt);
expect(commentReads).toHaveLength(1);
});
try {
const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] });
expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "selected wake canary" });
expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "selected wake canary" }] });
} finally {
inspectPreparation.delete(f.agentId);
select.mockRestore();
}
});
it("cancels deleted-only queued comments before adapter dispatch", async () => {
const f = await seed(), commentId = randomUUID();
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
authorType: "user", authorUserId: "context-user", body: "discarded wake", deletedAt: new Date() });
const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented",
payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] } });
expect(run).not.toBeNull();
await heartbeat.drainActiveRunExecutions();
const finished = await heartbeat.getRun(run!.id);
expect(finished).toMatchObject({ status: "cancelled", error: "Queued messages were discarded before dispatch" });
expect(captured.has(run!.id)).toBe(false);
});
it("preserves deleted comment metadata without delivering its body", async () => {
const f = await seed();
const commentId = randomUUID();
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
authorType: "user", authorUserId: "context-user", body: "deleted wake canary", deletedAt: new Date(),
presentation: { fixture: "deleted" }, metadata: { fixture: "deleted" } });
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "running", issueId: f.issueId }).returning();
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
await prepareHeartbeatTaskContext(db, {
run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
});
expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "", presentation: null, metadata: null });
expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "", presentation: null, metadata: null }] });
expect(context.paperclipTaskMarkdown).not.toContain("deleted wake canary");
});
it.each([false, true])("uses admitted skill-test scope when recovery is %s", async (recovering) => {
const f = await seed();
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "running", issueId: f.issueId }).returning();
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
const context: Record<string, unknown> = { issueId: f.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } };
const pinned = vi.fn(async () => null);
const preparing = prepareHeartbeatTaskContext(db, {
run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
policy: { recovering, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: pinned },
});
if (recovering) await expect(preparing).rejects.toThrow("skill test's saved version is unavailable");
else await preparing;
expect(pinned).toHaveBeenCalledTimes(recovering ? 1 : 0);
expect(context.paperclipAgentJwtScope).toEqual({ kind: recovering ? "skill_test" : "standard", ...(recovering ? { issueId: f.issueId } : {}) });
});
it("retains the context issue scope when no issue reference is available", async () => {
const f = await seed();
const otherIssueId = randomUUID(), commentId = randomUUID();
await db.insert(issues).values({ id: otherIssueId, companyId: f.companyId, title: "Other issue", issueNumber: 2 });
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: otherIssueId,
authorType: "user", authorUserId: "context-user", body: "other issue canary" });
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "running", issueId: f.issueId }).returning();
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
await prepareHeartbeatTaskContext(db, {
run, agent, task: { issueRef: null, issueContext: null, context, taskSession: null },
policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
});
expect(JSON.stringify(context.paperclipWake)).not.toContain("other issue canary");
expect(context.paperclipWakeComment).toBeUndefined();
});
it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => {
const f = await seed();
const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId });
@@ -1,3 +1,5 @@
import { subscribeDatabaseWork } from "@paperclipai/db";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { execFile } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
@@ -888,7 +890,7 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {
const tempRoots: string[] = [];
beforeAll(async () => {
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl);
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl, { maxConnections: 6 });
else {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-branch-containment-");
db = createDb(tempDb.connectionString);
@@ -1173,15 +1175,15 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {
},
});
const routineEvents: string[] = [];
const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event));
const heartbeat = heartbeatService(db);
const run = await heartbeat.wakeup(agentId, {
source: "assignment",
triggerDetail: "system",
reason: "issue_assigned",
payload: { issueId },
contextSnapshot: { issueId, wakeReason: "issue_assigned" },
});
let run;
try {
run = await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned",
payload: { issueId }, contextSnapshot: { issueId, wakeReason: "issue_assigned" } });
} finally { unsubscribeRoutine(); }
expect(routineEvents).toContain("intent"); expect(routineEvents).toContain("settled");
expect(run).toBeNull();
expect(adapterExecute).not.toHaveBeenCalled();
@@ -1,8 +1,9 @@
import { randomUUID } from "node:crypto";
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db";
import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
@@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
(support.supported ? describe : describe.skip)("historical cleanup verification", () => {
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000);
afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true);
const state = async () => (await read()).backgroundWork;
@@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }),
});
}
it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => {
const f = await seed(adHoc);
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value);
let absent = false;
const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => {
if (!absent) return Response.json({ absent: false });
const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000);
expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id,
leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId });
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url");
const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1",
sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url");
return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` });
});
vi.stubGlobal("fetch", fetchImpl);
const runtime = environmentRuntimeService(db);
const service = historicalCleanupVerificationService(db, {
inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined,
});
expect(await state()).toBe("present");
expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 });
expect(await state()).toBe("present");
absent = true;
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
expect(await state()).toBe("none");
const [current] = await db.select().from(environmentLeases);
expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId });
expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination");
expect(await service.sweep()).toMatchObject({ checked: 0 });
expect(fetchImpl).toHaveBeenCalledTimes(2);
});
it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => {
await seed(true);
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value);
const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl);
const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null,
inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }),
}] });
const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined });
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 });
expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled();
});
it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => {
const f = await seed(adHoc); expect(await state()).toBe("present");
const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
@@ -8,6 +8,7 @@ import {
approvals,
companies,
createDb,
applyPendingMigrations,
heartbeatRuns,
issueApprovals,
issueRelations,
@@ -21,7 +22,10 @@ import {
import { issueService } from "../services/issues.js";
import { buildIssueGraphLivenessIncidentKey } from "../services/recovery/origins.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_BLOCKER_ATTENTION_TEST_DATABASE_URL");
const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
if (!embeddedPostgresSupport.supported) {
@@ -36,8 +40,9 @@ describeEmbeddedPostgres("issue blocker attention", () => {
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-");
db = createDb(tempDb.connectionString);
if (externalUrl) await applyPendingMigrations(externalUrl);
else tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-");
db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 });
svc = issueService(db);
}, 20_000);
@@ -55,6 +60,7 @@ describeEmbeddedPostgres("issue blocker attention", () => {
});
afterAll(async () => {
await db?.$client.end({ timeout: 1 });
await tempDb?.cleanup();
});
@@ -0,0 +1,382 @@
import { setImmediate as immediate } from "node:timers/promises";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { heartbeatService } from "../services/heartbeat.js";
import { issueService } from "../services/issues.js";
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
import { registerIssueRecoveryWork } from "../services/recovery/work.js";
import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
import { taskWatchdogService } from "../services/task-watchdogs.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
vi.setConfig({ testTimeout: 90_000 });
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("issue recovery production producers and deadlines", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
let heartbeat: ReturnType<typeof heartbeatService>;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
let worker: ReturnType<ReturnType<typeof heartbeatService>["registerIssueStateRecoveryWork"]>;
let dispatch: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]>;
let cancellation: ReturnType<ReturnType<typeof heartbeatService>["registerScheduledRetryCancellationWork"]>;
let draining = false;
const tasks = new Map<string, { nextRunAt?: () => number | null; hasPending: () => Promise<boolean> }>();
const errors: unknown[] = [];
const passes = new Set<Promise<unknown>>();
const extraStops: Array<() => Promise<void>> = [];
const extraDeadlines: Array<() => number | null> = [];
beforeAll(async () => {
if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-");
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
}, 90_000);
beforeEach(async () => {
await db.execute(sql`TRUNCATE companies CASCADE`);
heartbeat = heartbeatService(db); draining = false; errors.length = 0;
});
afterEach(async () => {
for (const stop of extraStops.splice(0)) await stop();
extraDeadlines.length = 0;
await worker?.stop(); await dispatch?.stop(); await cancellation?.stop(); await coordinator?.stop();
await heartbeat.drainActiveRunExecutions();
vi.useRealTimers(); vi.restoreAllMocks();
expect(errors).toEqual([]);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
async function start() {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
onError: error => errors.push(error) });
tasks.clear();
const register = coordinator.register.bind(coordinator);
coordinator.register = (queue, task) => {
tasks.set(queue, task);
return register(queue, { ...task, run: signal => {
const work = task.run(signal); passes.add(work);
void work.then(() => passes.delete(work), () => passes.delete(work));
return work;
} });
};
worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining);
dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining);
await Promise.all([worker.ready, dispatch.ready, cancellation.ready]);
}
async function compete() {
const other = heartbeatService(db);
const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
const work = other.registerIssueStateRecoveryWork(competing, () => true);
extraStops.push(() => competing.stop()); extraDeadlines.push(competing.nextWakeAt); await work.ready;
}
async function fixture() {
const companyId = randomUUID();
await db.insert(authUsers).values({ id: "recovery-user", name: "Recovery operator", email: "recovery-user@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 7)}`,
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "recovery-user" });
await db.insert(companyMemberships).values({ companyId, principalId: "recovery-user", principalType: "user", status: "active", membershipRole: "owner" });
const [agent] = await db.insert(agents).values({ companyId, name: "Recovery process", role: "engineer", status: "idle",
adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('recovery-proof')"], cwd: process.cwd() },
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
}).returning();
return { companyId, agentId: agent.id };
}
async function settled() {
await vi.waitFor(async () => {
await heartbeat.drainActiveRunExecutions();
const wake = coordinator.nextWakeAt();
const detail = wake === null ? "" : JSON.stringify({ now: Date.now(), tasks: await Promise.all([...tasks].map(async ([queue, task]) => ({ queue, at: task.nextRunAt?.(), pending: await task.hasPending() }))),
runs: (await db.select().from(heartbeatRuns)).map(run => ({ id: run.id, status: run.status, error: run.errorCode, lease: run.controllerLeaseExpiresAt, reason: run.contextSnapshot?.wakeReason })) });
expect(wake, detail).toBeNull();
expect(extraDeadlines.map(next => next())).toEqual(extraDeadlines.map(() => null));
expect(idleWorkSnapshot().active).toBe(0);
}, { timeout: 40_000 });
}
async function quiet() {
await settled();
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute");
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); }
finally { vi.useRealTimers(); reads.mockRestore(); executes.mockRestore(); }
}
async function finish(issueId: string) {
await issueService(db).update(issueId, { status: "done" });
// Closing one blocker can legitimately start its dependent. Wait for this
// task's dispatch and execution; quiet() verifies the whole fixture later.
await vi.waitFor(async () => {
await Promise.allSettled([...passes]);
await heartbeat.drainActiveRunExecutions();
expect(await db.select().from(heartbeatRuns).where(and(
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
sql`${heartbeatRuns.status} in ('queued', 'running')`,
))).toHaveLength(0);
}, { timeout: 15_000 });
const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
if (saved.length > 0) {
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
// Keep PostgreSQL I/O and coordinator settlement ahead of virtual time.
// A single leap can otherwise occur before the retry deadline is armed.
while (Date.now() <= due) {
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
await Promise.allSettled([...passes]);
await immediate();
}
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(500);
await Promise.allSettled([...passes]);
await heartbeat.drainActiveRunExecutions();
const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
expect(remaining).toHaveLength(0);
});
for (const retry of saved) {
const [settledRetry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retry.id));
expect(settledRetry.status).toBe("cancelled");
}
}
}
async function completed(issueId: string) {
let runId: string | undefined;
await vi.waitFor(async () => {
const runs = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`);
const run = runs.find(row => row.status === "running" || row.status === "succeeded");
expect(run).toBeDefined(); runId = run!.id;
}, { timeout: 15_000 });
await vi.waitFor(async () => {
await heartbeat.drainActiveRunExecutions();
const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
expect(current.status, JSON.stringify({ error: current.error, code: current.errorCode, stderr: current.stderrExcerpt })).toBe("succeeded");
}, { timeout: 15_000 });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
expect(run.stdoutExcerpt).toContain("recovery-proof");
return run;
}
it("keeps empty, configured idle, and terminal history quiet", async () => {
await fixture(); await start(); await quiet();
});
it("creates a blocking child through the ordinary root service transaction", async () => {
const f = await fixture();
const parent = await issueService(db).create(f.companyId, { title: "Parent", status: "todo" });
const child = await issueService(db).createChild(parent.id, { title: "Blocking child", status: "todo", blockParentUntilDone: true });
expect(child.parentBlockerAdded).toBe(true);
expect(await db.select().from(issueRelations).where(and(eq(issueRelations.companyId, f.companyId),
eq(issueRelations.issueId, child.issue.id), eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1);
});
it.each(["startup", "committed"])("restores blocker-only and completed-action debt with no assignment candidates (%s)", async mode => {
const f = await fixture();
if (mode === "committed") { await start(); await quiet(); }
const blocker = await issueService(db).create(f.companyId, { title: "Orphan blocker", status: "blocked", createdByAgentId: f.agentId });
const dependent = await issueService(db).create(f.companyId, { title: "Blocked dependent", status: "blocked", blockedByIssueIds: [blocker.id] });
const completedSource = await issueService(db).create(f.companyId, { title: "Completed source", status: "done" });
const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: completedSource.id,
kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
fingerprint: "source-terminal", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
if (mode === "startup") await start();
await vi.waitFor(async () => {
expect((await db.select().from(issues).where(eq(issues.id, blocker.id)))[0].assigneeAgentId).toBe(f.agentId);
expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
}, { timeout: 40_000 });
await finish(blocker.id); await finish(dependent.id); await quiet();
});
it("bounds subsidiary scans for100 assignment candidates and coalesces repeated company writes", async () => {
const f = await fixture(), foreign = await fixture();
await db.update(agents).set({ lifecycleState: "preparing" }).where(eq(agents.id, f.agentId));
await db.insert(issues).values(Array.from({ length: 100 }, (_, index) => ({ companyId: f.companyId,
title: `Waiting setup ${index}`, status: "todo", assigneeAgentId: f.agentId })));
const [source] = await db.insert(issues).values({ companyId: f.companyId, title: "Done source", status: "done" }).returning();
const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: source.id,
kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
fingerprint: "query-budget", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
const scans = { blockers: 0, actions: 0, loads: 0 };
const originalSelect = db.select.bind(db);
vi.spyOn(db, "select").mockImplementation(((fields?: Record<string, unknown>) => {
const query = originalSelect(fields as never); const from = query.from.bind(query);
query.from = ((table: unknown) => {
if (table === issues && fields?.id === issues.id && fields.companyId === issues.companyId && Object.keys(fields).length === 2) scans.loads++;
if (table === issueRelations && fields?.createdByAgentId === issues.createdByAgentId) scans.blockers++;
if (table === issueRecoveryActions && fields?.action === issueRecoveryActions) scans.actions++;
return from(table as never);
}) as typeof query.from;
return query;
}) as typeof db.select);
await start(); await settled();
expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(8);
expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
const [foreignSource] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign done", status: "done" }).returning();
const [foreignAction] = await db.insert(issueRecoveryActions).values({ companyId: foreign.companyId, sourceIssueId: foreignSource.id,
kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "foreign-query-budget", nextAction: "Inspect" }).returning();
const before = { ...scans };
for (let index = 0; index < 100; index++) await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
await new Promise(resolve => setTimeout(resolve, 100));
expect(scans.loads).toBe(before.loads);
await settled();
expect(scans.actions).toBe(before.actions); expect(scans.blockers).toBe(before.blockers);
expect(scans.loads).toBeLessThanOrEqual(before.loads + 4);
expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active");
expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet();
});
it("limits an ordinary comment to its issue among one thousand open assignments", async () => {
const f = await fixture();
const rows = await db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({
companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId,
}))).returning({ id: issues.id });
const checked: string[] = [];
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
const register = coordinator.register.bind(coordinator);
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
const pass = task.run(signal); passes.add(pass);
void pass.then(() => passes.delete(pass), () => passes.delete(pass));
return pass;
} });
// Observe the actual paged reconstruction boundary without executing one
// thousand adapters. The producer, transaction, coordinator and SQL are real;
// the other acceptance cases exercise the resulting recovery decisions.
worker = registerIssueRecoveryWork(db, coordinator, {
stranded: async scope => { checked.push(scope.issueId); },
orphanBlockers: async () => {}, recoveryActions: async () => {},
dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {},
watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false,
}, () => true);
await worker.ready;
async function settleWork() {
for (let step = 0; step < 120; step++) {
await vi.advanceTimersByTimeAsync(1000);
await Promise.allSettled([...passes]); await immediate();
if (coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return;
}
throw new Error("Recovery did not settle its committed work within two virtual minutes");
}
await settleWork();
expect(new Set(checked).size).toBe(1000);
checked.length = 0;
await issueService(db).addComment(rows[0].id, "New evidence on this task", { userId: "recovery-user" });
await settleWork();
expect(checked).toEqual([rows[0].id]);
// Company eligibility changes still reconstruct all assignments. Narrowing
// comment hints must not hide unrelated work after an operator resumes it.
checked.length = 0;
await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
await settleWork();
expect(new Set(checked).size).toBe(1000);
await quiet();
});
it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => {
const f = await fixture(); await start(); await quiet();
const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId });
await completed(issue.id);
await finish(issue.id); await quiet();
}, 90_000);
it("reacts to blocker resolution once and dispatches the durable dependency wake", async () => {
const f = await fixture();
const blocker = await issueService(db).create(f.companyId, { title: "Blocker", status: "todo" });
const dependent = await issueService(db).create(f.companyId, { title: "Dependent", status: "blocked", assigneeAgentId: f.agentId,
blockedByIssueIds: [blocker.id] });
await start(); await quiet();
await compete();
await issueService(db).update(blocker.id, { status: "done" });
const run = await completed(dependent.id);
expect(run.contextSnapshot?.wakeReason).toBe("issue_blockers_resolved");
await finish(dependent.id); await quiet();
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_blockers_resolved'`))).toHaveLength(1);
}, 90_000);
it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => {
const f = await fixture();
const source = await issueService(db).create(f.companyId, { title: "Finished subtree", status: "done" });
await start(); await quiet();
await compete();
const watchdog = await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
let reviewId: string | undefined;
await vi.waitFor(async () => {
const [row] = await db.select().from(issueWatchdogs).where(eq(issueWatchdogs.id, watchdog.watchdog.id));
expect(row.watchdogIssueId).toBeTruthy(); reviewId = row.watchdogIssueId!;
}, { timeout: 15_000 });
await completed(reviewId!);
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1);
await finish(reviewId!);
await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
}, 90_000);
it("bounds ordinary writes to their company and reconstructs foreign stale debt after restart", async () => {
const selected = await fixture(); await start(); await quiet();
const foreign = await fixture();
const [run] = await db.insert(heartbeatRuns).values({ companyId: foreign.companyId, agentId: foreign.agentId,
status: "failed", finishedAt: new Date() }).returning();
const [stale] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign stale lock", status: "done",
checkoutRunId: run.id }).returning();
await issueService(db).create(selected.companyId, { title: "Selected change", status: "done" }); await quiet();
expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBe(run.id);
await worker.stop(); await dispatch.stop(); await cancellation.stop(); await coordinator.stop();
heartbeat = heartbeatService(db); await start(); await quiet();
expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBeNull();
});
it("fences nested assignment writes until outer commit and ignores rollback", async () => {
const f = await fixture(); await start(); await quiet();
let committedId = "";
await db.transaction(async outer => {
await outer.transaction(async nested => {
const issue = await issueService(db).create(f.companyId, { title: "Nested commit", status: "todo", assigneeAgentId: f.agentId }, nested);
committedId = issue.id;
});
await new Promise(resolve => setTimeout(resolve, 100));
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
});
await completed(committedId); await finish(committedId); await settled();
await expect(db.transaction(async outer => {
await outer.transaction(async nested => {
await issueService(db).create(f.companyId, { title: "Nested rollback", status: "todo", assigneeAgentId: f.agentId }, nested);
});
throw new Error("rollback control");
})).rejects.toThrow("rollback control");
await quiet();
expect(await db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0);
}, 90_000);
it("retains the first committed assignment during drain and executes on resume", async () => {
const f = await fixture(); await start(); await quiet(); draining = true;
const issue = await issueService(db).create(f.companyId, { title: "Drained assignment", status: "todo", assigneeAgentId: f.agentId });
await new Promise(resolve => setTimeout(resolve, 100));
const reads = vi.spyOn(db, "select");
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
try { await vi.advanceTimersByTimeAsync(90_000); expect(reads).not.toHaveBeenCalled(); }
finally { vi.useRealTimers(); reads.mockRestore(); }
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
draining = false; worker.wake(); dispatch.wake();
await completed(issue.id); await finish(issue.id); await quiet();
}, 90_000);
it("retains a fresh subtree's exact first-run grace then disarms on disable", async () => {
const f = await fixture();
const source = await issueService(db).create(f.companyId, { title: "Fresh unassigned subtree", status: "todo" });
await start(); await quiet();
await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(source.createdAt.getTime() + 15_000), { timeout: 5_000 });
expect(await db.select().from(issues).where(eq(issues.originKind, "task_watchdog"))).toHaveLength(0);
await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
});
it("waits for a live foreign controller's exact stored lease before clearing a terminal lock", async () => {
const f = await fixture();
const source = await issueService(db).create(f.companyId, { title: "Remote owner", status: "done" });
const lease = new Date(Date.now() + 5_000);
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running",
runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: lease,
contextSnapshot: { issueId: source.id }, processPid: 2_000_000_000, startedAt: new Date() }).returning();
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, source.id));
await start();
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.getTime()), { timeout: 3_000 });
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running");
await vi.waitFor(async () => expect((await db.select().from(issues).where(eq(issues.id, source.id)))[0].executionRunId).toBeNull(), { timeout: 12_000 });
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded");
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
}, 90_000);
});
@@ -0,0 +1,149 @@
import { PgDialect } from "drizzle-orm/pg-core";
import { afterEach, describe, expect, it, vi } from "vitest";
import { issueRecoveryActions, issueWatchdogs } from "@paperclipai/db";
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
import { registerIssueRecoveryWork } from "../services/recovery/work.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
const stops: Array<() => Promise<void>> = [];
afterEach(async () => { for (const stop of stops.splice(0)) await stop(); vi.useRealTimers(); });
function fixture() {
vi.useFakeTimers(); vi.setSystemTime(1_000_000);
const dialect = new PgDialect();
const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }>, blockers: [] as Array<{ id: string; companyId: string }>, actions: [] as Array<{ id: string; companyId: string }> };
const execute = vi.fn(async (query: any) => {
if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
throw new Error("unexpected SQL");
});
const load = vi.fn((table: unknown, distinct = false) => distinct
? table === issueRecoveryActions ? rows.actions : rows.blockers
: table === issueWatchdogs ? rows.watchdogs : rows.issues);
const query = (distinct = false) => ({ from: (table: unknown) => ({ where: () => ({ orderBy: () => ({ limit: async () => load(table, distinct) }) }) }) });
const db: any = installDatabaseWorkSignals({ execute,
select: () => query(), selectDistinct: () => query(true),
transaction: async (callback: (tx: any) => Promise<unknown>) => callback({ execute, select: db.select, selectDistinct: db.selectDistinct, transaction: db.transaction }),
});
const state = { draining: false, standby: false, suppressed: false, deadline: null as number | null, fail: false, incomplete: false };
const check = vi.fn(async (scope: { onDeferredDeadline(at: number): void }) => {
if (state.fail) throw new Error("database unavailable");
if (state.deadline !== null && state.deadline > Date.now()) scope.onDeferredDeadline(state.deadline);
});
const orphanBlockers = vi.fn(async () => {}), recoveryActions = vi.fn(async () => {});
const suppression = vi.fn(async () => state.suppressed);
const owner = { orphanBlockers, recoveryActions, stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; },
locks: check, watchdogs: async (scope: any) => { await check(scope); return { incomplete: state.incomplete ? 1 : 0 }; },
schedulingSuppressed: suppression };
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.standby,
canReconcile: () => !state.standby, onError: vi.fn() });
const worker = registerIssueRecoveryWork(db, coordinator, owner, () => !state.draining && !state.standby);
stops.push(() => coordinator.stop());
return { db, rows, state, check, load, coordinator, worker, orphanBlockers, recoveryActions, suppression,
notify: () => db.transaction((tx: any) => notifyIssueRecovery(tx, { companyId: "selected" })),
settle: () => vi.advanceTimersByTimeAsync(1_000) };
}
describe("issue recovery obligation lifecycle", () => {
it("has no timer, SQL, or owner probe after empty reconstruction", async () => {
const f = fixture(); await f.worker.ready;
expect(f.coordinator.nextWakeAt()).toBeNull(); const loaded = f.load.mock.calls.length;
await vi.advanceTimersByTimeAsync(20 * 60_000);
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
});
it("reads scheduling suppression once per pass instead of once per issue", async () => {
const f = fixture();
f.rows.issues.push(...Array.from({ length: 25 }, (_, index) => ({ id: `issue-${index}`, companyId: "selected" })));
await f.worker.ready;
expect(f.check).toHaveBeenCalledTimes(75);
expect(f.suppression).toHaveBeenCalledTimes(6);
expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("backs off a failed suppression read without probing an empty or failed owner repeatedly", async () => {
const f = fixture(); f.suppression.mockRejectedValueOnce(new Error("settings unavailable"));
await f.worker.ready; const reads = f.suppression.mock.calls.length;
expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
await vi.advanceTimersByTimeAsync(29_999); expect(f.suppression).toHaveBeenCalledTimes(reads);
await vi.advanceTimersByTimeAsync(1); expect(f.suppression).toHaveBeenCalledTimes(reads + 1);
expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("normalizes a wide scope with cyclic SQL objects before publishing work hints", async () => {
const f = fixture(); await f.worker.ready;
const wide = { companyId: "selected", endpointId: issueRecoveryActions.id, table: issueRecoveryActions };
await f.db.transaction((tx: object) => notifyIssueRecovery(tx, wide)); await f.settle();
expect(f.coordinator.nextWakeAt()).toBeNull();
const loads = f.load.mock.calls.length;
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.load).toHaveBeenCalledTimes(loads);
});
it("runs orphan blocker and action duties without assignment entries", async () => {
const f = fixture(); f.rows.blockers.push({ id: "selected", companyId: "selected" });
f.rows.actions.push({ id: "selected", companyId: "selected" }); await f.worker.ready;
expect(f.check).not.toHaveBeenCalled();
expect(f.orphanBlockers).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
expect(f.recoveryActions).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("coalesces repeated company reconstruction without delaying an existing lease deadline", async () => {
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
f.state.deadline = 1_015_000; await f.worker.ready;
await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length, checked = f.check.mock.calls.length;
for (let index = 0; index < 100; index++) { await f.notify(); await vi.advanceTimersByTimeAsync(10); }
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).toHaveBeenCalledTimes(checked);
await vi.advanceTimersByTimeAsync(13_000);
expect(f.state.deadline).toBe(1_015_000);
expect(f.coordinator.nextWakeAt()).toBe(1_031_000);
await vi.advanceTimersByTimeAsync(16_000);
expect(f.load.mock.calls.length).toBeGreaterThan(loaded);
expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("retains exact grace and lease deadlines without early owner probes", async () => {
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.deadline = 1_015_000;
await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_015_000);
const checked = f.check.mock.calls.length;
await vi.advanceTimersByTimeAsync(14_999); expect(f.check).toHaveBeenCalledTimes(checked);
await vi.advanceTimersByTimeAsync(1); expect(f.check.mock.calls.length).toBe(checked * 2);
expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("releases settled producer holds during domain drain while coordinator remains open", async () => {
const f = fixture(); await f.worker.ready; f.state.draining = true;
f.rows.issues.push({ id: "issue", companyId: "selected" });
await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length;
await vi.advanceTimersByTimeAsync(90_000);
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
expect(idleWorkSnapshot().active).toBe(0);
f.state.draining = false; f.worker.wake(); await f.settle();
expect(f.check).toHaveBeenCalled(); expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("retries concrete failed entries and then disarms without reloading the company", async () => {
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); f.state.fail = true;
await f.worker.ready; const loaded = f.load.mock.calls.length;
expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
await vi.advanceTimersByTimeAsync(29_999); const checked = f.check.mock.calls.length;
f.state.fail = false; await vi.advanceTimersByTimeAsync(1);
expect(f.check.mock.calls.length).toBe(checked * 2); expect(f.load).toHaveBeenCalledTimes(loaded);
expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("keeps an incomplete watchdog subtree as concrete retry debt", async () => {
const f = fixture(); f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.incomplete = true;
await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
const loaded = f.load.mock.calls.length; f.state.incomplete = false;
await vi.advanceTimersByTimeAsync(30_000);
expect(f.check).toHaveBeenCalledTimes(2); expect(f.load).toHaveBeenCalledTimes(loaded);
expect(f.coordinator.nextWakeAt()).toBeNull();
});
it("keeps a concurrent producer dirty until the in-flight owner has completed", async () => {
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); await f.worker.ready;
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
f.check.mockImplementationOnce(() => held);
await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await vi.advanceTimersByTimeAsync(30_000);
expect(f.coordinator.nextWakeAt()).toBeNull();
expect(f.check.mock.calls.length).toBeGreaterThan(3);
});
it("retains startup work on standby and cancels future deadlines on shutdown", async () => {
const f = fixture(); f.state.standby = true; f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" });
await f.worker.ready; await vi.advanceTimersByTimeAsync(90_000); expect(f.load).not.toHaveBeenCalled();
f.state.standby = false; f.state.deadline = Date.now() + 60_000; f.worker.wake(); await f.settle();
expect(f.check).toHaveBeenCalledOnce(); await f.worker.stop();
await vi.advanceTimersByTimeAsync(120_000); expect(f.check).toHaveBeenCalledOnce();
});
});
@@ -41,7 +41,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
beforeAll(async () => {
if (externalUrl) {
await applyPendingMigrations(externalUrl);
db = createDb(externalUrl);
db = createDb(externalUrl, { maxConnections: 6 });
return;
}
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-");
+1 -1
View File
@@ -4151,7 +4151,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
beforeAll(async () => {
if (!dependencyTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-blockers-");
db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString);
db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
svc = issueService(db);
await ensureIssueRelationsTable(db);
}, 20_000);
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { inArray } from "drizzle-orm";
@@ -131,7 +132,8 @@ describe("merged pull-request confirmation extraction", () => {
});
});
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_SWEEP_TEST_DATABASE_URL");
const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
describeEmbeddedPostgres("merged pull-request confirmation sweep", () => {
@@ -139,8 +141,8 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => {
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-");
db = createDb(tempDb.connectionString);
if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-");
db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 });
}, 20_000);
afterEach(async () => {
@@ -153,6 +155,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => {
});
afterAll(async () => {
await db?.$client.end({ timeout: 1 });
await tempDb?.cleanup();
});
@@ -276,7 +279,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => {
repo: "paperclip",
number: 39,
}])).resolves.toEqual({
checked: 6,
checked: 3,
candidates: 3,
accepted: 2,
woken: 2,
@@ -0,0 +1,366 @@
import { createServer } from "node:http";
import { createPullRequestMergeStateResolver } from "../services/github-pull-request-merge.js";
import { randomUUID } from "node:crypto";
import { setTimeout as delay } from "node:timers/promises";
import { eq, sql } from "drizzle-orm";
import { activityLog, agents, chatActions, chatConversations, chatEndpoints, chatPublications, companies, createDb, issues, issueThreadInteractions, toolApplications, toolConnections, type Db } from "@paperclipai/db";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { issueThreadInteractionService } from "../services/issue-thread-interactions.js";
import { issueService } from "../services/issues.js";
import { registerMergeConfirmationWork, MERGE_CONFIRMATION_RECONCILE_MS } from "../services/merged-pr-confirmation-work.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_WORK_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("merge confirmation work with PostgreSQL and simulated GitHub", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
let companyId: string, issueId: string, agentId: string;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
let draining = false, standby = false;
const errors: unknown[] = [];
const provider = vi.fn(async (_companyId: string, _reference: { number: number }): Promise<"open" | "merged" | "unknown"> => "merged");
const wakeup = vi.fn(async () => ({ id: "simulated-durable-wake" }));
beforeAll(async () => {
if (!external) temporary = await startEmbeddedPostgresTestDatabase("merged-pr-work-");
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
}, 30_000);
beforeEach(async () => {
errors.length = 0; draining = false; standby = false;
provider.mockReset().mockResolvedValue("merged"); wakeup.mockClear();
companyId = randomUUID(); agentId = randomUUID(); issueId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Merge worker", issuePrefix: `M${companyId.slice(0, 6)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "idle", adapterType: "codex_local" });
await db.insert(issues).values({ id: issueId, companyId, title: "Merge task", status: "todo", assigneeAgentId: agentId });
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false });
});
afterEach(async () => {
await coordinator?.stop(); coordinator = undefined;
vi.useRealTimers(); vi.restoreAllMocks();
await db.delete(chatActions); await db.delete(chatPublications); await db.delete(chatConversations); await db.delete(chatEndpoints);
await db.delete(toolConnections); await db.delete(toolApplications);
await db.delete(activityLog); await db.delete(issueThreadInteractions); await db.delete(issues);
await db.delete(agents); await db.delete(companies);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
const service = (database: Db = db) => issueThreadInteractionService(database, { resolvePullRequestState: provider, wakeup });
const input = (number = 39) => ({ kind: "request_confirmation" as const, title: `Merge paperclipai/paperclip#${number}?`,
resolverPolicy: "human_only" as const, continuationPolicy: "wake_assignee_on_accept" as const,
payload: { version: 1 as const, prompt: "Merge the PR?" } });
const create = (database: Db = db, number = 39) => service(database).create({ id: issueId, companyId }, input(number), { agentId });
const stored = (id: string) => db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, id)).then(rows => rows[0]);
async function start() {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining && !standby,
canReconcile: () => !standby, onError: error => errors.push(error) });
const worker = registerMergeConfirmationWork(coordinator, service(), () => !draining && !standby);
await worker.ready;
return worker;
}
async function waitFor(check: () => Promise<void> | void) {
for (let attempt = 0; attempt < 100; attempt++) {
await vi.advanceTimersByTimeAsync(1000);
await delay(10);
try { await check(); return; } catch (error) { if (attempt === 99) throw error; }
}
}
const settle = () => waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
async function quiet() {
await waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); });
const reads = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute");
const calls = provider.mock.calls.length;
await vi.advanceTimersByTimeAsync(3 * MERGE_CONFIRMATION_RECONCILE_MS);
expect(reads).not.toHaveBeenCalled(); expect(transactions).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled();
expect(provider).toHaveBeenCalledTimes(calls);
reads.mockRestore(); transactions.mockRestore(); executes.mockRestore();
}
it("settles empty and configured idle, then ordinary creation accepts, audits, wakes once and becomes quiet with retained history", async () => {
await start(); await quiet(); expect(provider).not.toHaveBeenCalled();
const card = await create();
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted"));
await quiet();
expect(wakeup).toHaveBeenCalledTimes(1);
expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${card.id}:accepted`, allowRunCoalescing: false }));
expect(await db.select().from(activityLog)).toEqual(expect.arrayContaining([expect.objectContaining({
action: "issue.thread_interaction_accepted", actorId: "system:pr-merged", details: expect.objectContaining({ interactionId: card.id }),
})]));
expect(errors).toEqual([]);
});
it("reconciles an open candidate at its deadline, never early, then stops after merge", async () => {
provider.mockResolvedValue("open"); await start(); const card = await create();
await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle();
const dueAt = coordinator!.nextWakeAt()!;
expect(dueAt).toBeGreaterThan(Date.now());
await vi.advanceTimersByTimeAsync(dueAt - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1);
provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1);
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
expect(provider).toHaveBeenCalledTimes(2); expect(wakeup).toHaveBeenCalledTimes(1);
});
it("manual cancellation removes the candidate deadline without provider work", async () => {
provider.mockResolvedValue("open"); await start(); const card = await create();
await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle();
await service().withdrawInteraction({ id: issueId, companyId }, card.id, {}, { agentId });
await quiet(); expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).not.toHaveBeenCalled();
expect((await stored(card.id)).status).toBe("cancelled");
});
it("does not accept before caller-owned outer commit and forgets a rolled-back create", async () => {
await start(); await quiet();
let release!: () => void, created!: () => void;
const barrier = new Promise<void>(resolve => { release = resolve; });
const inserted = new Promise<void>(resolve => { created = resolve; });
let id = "";
const transaction = db.transaction(async tx => {
id = (await create(tx as unknown as Db)).id; created(); await barrier;
});
await inserted;
await vi.advanceTimersByTimeAsync(10_000); expect(provider).not.toHaveBeenCalled();
expect(await stored(id)).toBeUndefined(); release(); await transaction;
await waitFor(async () => expect((await stored(id)).status).toBe("accepted")); await quiet();
const prior = provider.mock.calls.length;
await expect(db.transaction(async tx => { await create(tx as unknown as Db, 41); throw new Error("rollback"); })).rejects.toThrow("rollback");
await quiet(); expect(provider).toHaveBeenCalledTimes(prior); expect(errors).toEqual([]);
});
it("retains provider failures across drain, resumes without overlap, and clears retry after success", async () => {
provider.mockRejectedValue(new Error("GitHub unavailable")); await start(); const card = await create();
await waitFor(() => expect(errors).toHaveLength(1)); await settle();
expect((await stored(card.id)).status).toBe("pending");
draining = true;
const reads = vi.spyOn(db, "select"), calls = provider.mock.calls.length;
await vi.advanceTimersByTimeAsync(120_000);
expect(reads).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(calls); reads.mockRestore();
draining = false; provider.mockResolvedValue("merged");
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
expect(wakeup).toHaveBeenCalledTimes(1);
});
it("keeps first work committed during drain and startup during standby until resume", async () => {
standby = true; await start(); const card = await create();
await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled();
standby = false; draining = true;
await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled();
draining = false;
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
});
it("reconstructs pending cards on restart and ignores terminal or hidden tasks", async () => {
provider.mockResolvedValue("open"); const card = await create(); await start();
await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle();
await coordinator!.stop(); coordinator = undefined; provider.mockResolvedValue("merged");
await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
const hidden = await db.transaction(async tx => {
const card = await create(tx as unknown as Db, 41);
await issueService(tx as unknown as Db).update(issueId, { hiddenAt: new Date() });
return card;
});
await quiet(); expect((await stored(hidden.id)).status).toBe("pending");
await issueService(db).update(issueId, { hiddenAt: null });
await waitFor(async () => expect((await stored(hidden.id)).status).toBe("accepted")); await quiet();
});
it("preserves a real chat binding's durable continuation outbox on automatic acceptance", async () => {
const applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID();
await db.insert(toolApplications).values({ id: applicationId, companyId, applicationKey: `chat:slack:${endpointId}`, name: "Slack", type: "chat", status: "active" });
await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Slack", uid: endpointId,
connectionPurpose: "channel", transport: "chat_sdk", status: "active", enabled: true });
await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, provider: "slack", publicId: endpointId,
assignedAgentId: agentId, status: "active" });
const [conversation] = await db.insert(chatConversations).values({ companyId, endpointId, issueId,
externalConversationId: "merge-channel", externalThreadId: "merge-thread", externalLabel: "Merge thread", state: "active" }).returning();
await start(); await quiet(); const card = await create();
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
expect(await db.select().from(chatActions).where(eq(chatActions.kind, "interaction_wakeup"))).toEqual([
expect.objectContaining({ companyId, endpointId, conversationId: conversation.id, providerActionId: `interaction_wakeup:${card.id}`,
payload: expect.objectContaining({ interactionId: card.id, issueId, agentId, interactionStatus: "accepted" }) }),
]);
expect(await db.select().from(chatPublications)).toEqual(expect.arrayContaining([
expect.objectContaining({ idempotencyKey: `interaction:${card.id}:${endpointId}`, state: "cancelled" }),
]));
expect(errors).toEqual([]);
});
it("does not lose a newer card created during provider observation or wake the superseded card", async () => {
let release!: () => void, observed!: () => void;
const barrier = new Promise<void>(resolve => { release = resolve; });
const observing = new Promise<void>(resolve => { observed = resolve; });
provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; });
await start(); const old = await create();
await vi.advanceTimersByTimeAsync(1000); await observing;
const current = await create(db, 41);
expect((await stored(old.id)).status).toBe("expired");
release();
await waitFor(async () => expect((await stored(current.id)).status).toBe("accepted")); await quiet();
expect(wakeup).toHaveBeenCalledTimes(1);
expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${current.id}:accepted` }));
expect(errors).toEqual([]);
});
it("keeps unknown provider results as retryable debt and does not act before retry", async () => {
provider.mockResolvedValue("unknown"); await start(); const card = await create();
await waitFor(() => expect(errors).toHaveLength(1));
const next = coordinator!.nextWakeAt()!;
await vi.advanceTimersByTimeAsync(next - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1);
provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1);
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
expect(provider).toHaveBeenCalledTimes(2);
});
it("honors terminal issues, pure merge URL targets and continuationPolicy none", async () => {
await start(); await quiet();
const target = await service().create({ id: issueId, companyId }, { ...input(), title: "Merge the linked PR?", continuationPolicy: "none",
payload: { version: 1, prompt: "Merge the PR?", target: { type: "custom", key: "github-pr-39", href: "https://github.com/paperclipai/paperclip/pull/39" } } }, { agentId });
await waitFor(async () => expect((await stored(target.id)).status).toBe("accepted")); await quiet();
expect(wakeup).not.toHaveBeenCalled();
provider.mockClear();
const terminal = await db.transaction(async tx => {
const card = await create(tx as unknown as Db, 41);
await issueService(tx as unknown as Db).update(issueId, { status: "cancelled" });
return card;
});
await quiet(); expect((await stored(terminal.id)).status).toBe("expired");
expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled();
});
it("rechecks hidden state under the acceptance lock when visibility changes during a provider read", async () => {
let release!: () => void, observed!: () => void;
const barrier = new Promise<void>(resolve => { release = resolve; });
const observing = new Promise<void>(resolve => { observed = resolve; });
provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; });
await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing;
await issueService(db).update(issueId, { hiddenAt: new Date() }); release(); await quiet();
expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled();
await issueService(db).update(issueId, { hiddenAt: null });
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]);
});
it("unrelated pending confirmations do not retain worker SQL or provider work", async () => {
await start(); await quiet();
await service().create({ id: issueId, companyId }, { ...input(), title: "Approve the plan?", payload: { version: 1, prompt: "Approve the plan?" } }, { agentId });
await quiet(); expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled();
expect(errors).toEqual([]);
});
it("awaits an in-flight observation at shutdown and reconstructs its unaccepted card on restart", async () => {
let release!: () => void, observed!: () => void;
const barrier = new Promise<void>(resolve => { release = resolve; });
const observing = new Promise<void>(resolve => { observed = resolve; });
provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; });
await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing;
let stopped = false;
const stopping = coordinator!.stop().then(() => { stopped = true; });
await delay(10); expect(stopped).toBe(false); release(); await stopping; coordinator = undefined;
expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled();
const reads = vi.spyOn(db, "select");
await vi.advanceTimersByTimeAsync(2 * MERGE_CONFIRMATION_RECONCILE_MS); expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]);
});
it.each(["headers", "body", "not-found probe", "public visibility probe"] as const)("cancels an actual unresponsive HTTP %s read promptly at stop", async phase => {
vi.useRealTimers();
const card = await create();
let observed!: () => void, closed!: () => void;
const requested = new Promise<void>(resolve => { observed = resolve; });
const disconnected = new Promise<void>(resolve => { closed = resolve; });
const signals: AbortSignal[] = [];
const server = createServer((request, response) => {
if (phase === "not-found probe" && request.url?.includes("/pulls/")) { response.writeHead(404).end(); return; }
response.on("close", closed);
if (phase === "body") { response.writeHead(200, { "content-type": "application/json" }); response.write('{"merged":'); }
observed();
});
await new Promise<void>(resolve => { server.listen(0, "127.0.0.1", resolve); });
const address = server.address();
if (!address || typeof address === "string") throw new Error("Expected test HTTP address");
const resolveState = createPullRequestMergeStateResolver(db, {
requestTimeoutMs: 10_000,
credentialProvider: () => phase === "public visibility probe" ? { token: "fixture", source: "server_env", secretName: null } : null,
fetch: (url, init) => { signals.push(init!.signal!); return fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init); },
});
const real = issueThreadInteractionService(db, { resolvePullRequestState: resolveState, wakeup });
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
try {
const startupAt = performance.now();
const worker = registerMergeConfirmationWork(coordinator, real, () => true);
expect(performance.now() - startupAt).toBeLessThan(100);
await Promise.race([requested, delay(2000).then(() => { throw new Error("HTTP observation did not start"); })]);
const stoppingAt = performance.now();
await coordinator.stop(); coordinator = undefined;
expect(performance.now() - stoppingAt).toBeLessThan(1000);
await worker.ready;
await Promise.race([disconnected, delay(1000).then(() => { throw new Error("HTTP request remained open after stop"); })]);
expect(signals.at(-1)?.aborted).toBe(true);
expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]);
} finally {
await coordinator?.stop(); coordinator = undefined;
server.closeAllConnections(); await new Promise<void>(resolve => { server.close(() => resolve()); });
}
});
it("times out a real unresponsive HTTP read and recovers its durable candidate on restart", async () => {
vi.useRealTimers(); const card = await create();
let serveMerge = false, closed!: () => void;
const disconnected = new Promise<void>(resolve => { closed = resolve; });
const server = createServer((_request, response) => {
if (serveMerge) { response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ merged: true, state: "closed" })); return; }
response.on("close", closed);
});
await new Promise<void>(resolve => { server.listen(0, "127.0.0.1", resolve); });
const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected test HTTP address");
const real = issueThreadInteractionService(db, { resolvePullRequestState: createPullRequestMergeStateResolver(db, {
requestTimeoutMs: 100, tokenProvider: () => null,
fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init),
}), wakeup });
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
try {
const startedAt = performance.now();
await registerMergeConfirmationWork(coordinator, real, () => true).ready;
expect(performance.now() - startedAt).toBeLessThan(2000);
await Promise.race([disconnected, delay(1000).then(() => { throw new Error("Timed out HTTP request remained open"); })]);
expect(errors).toHaveLength(1); expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled();
expect(coordinator.nextWakeAt()).not.toBeNull();
await coordinator.stop(); coordinator = undefined; serveMerge = true;
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
await registerMergeConfirmationWork(coordinator, real, () => true).ready;
expect((await stored(card.id)).status).toBe("accepted"); expect(wakeup).toHaveBeenCalledTimes(1);
} finally {
await coordinator?.stop(); coordinator = undefined;
server.closeAllConnections(); await new Promise<void>(resolve => { server.close(() => resolve()); });
}
});
it("pages beyond a full pending-card page without duplicating the merge parser in SQL", async () => {
const card = await create();
const id = BigInt(`0x${card.id.replaceAll("-", "")}`);
await db.insert(issueThreadInteractions).values(Array.from({ length: 100 }, (_, index) => {
const hex = (id - BigInt(index + 1)).toString(16).padStart(32, "0");
return { id: `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20)}`,
companyId, issueId, kind: "request_confirmation", title: "Approve the plan?", payload: { version: 1 as const, prompt: "Approve the plan?" } };
}));
await start(); expect(provider).not.toHaveBeenCalled();
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]);
});
it("limits normalized merge hints to the owning company and matching PR and tolerates duplicates", async () => {
provider.mockResolvedValue("open"); const own = await create();
const foreignCompany = randomUUID(), foreignIssue = randomUUID();
await db.insert(companies).values({ id: foreignCompany, name: "Other company", issuePrefix: "OTH" });
await db.insert(issues).values({ id: foreignIssue, companyId: foreignCompany, title: "Other", status: "todo" });
const foreign = await service().create({ id: foreignIssue, companyId: foreignCompany }, input(), { userId: "board" });
const local = service();
const hint = [{ companyId, owner: "PAPERCLIPAI", repo: "paperclip", number: 39 }];
expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(1);
expect((await stored(own.id)).status).toBe("accepted"); expect((await stored(foreign.id)).status).toBe("pending");
expect(provider).not.toHaveBeenCalled();
expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(0); expect(wakeup).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,314 @@
import { issueTreeControlService } from "../services/issue-tree-control.js";
import { writeExecutionControl } from "../services/execution-control-notifications.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, heartbeatRuns, issues, pluginManagedResources, plugins, routineRuns, type Db } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { notifyRoutineExecutionStatus } from "../services/routine-execution-status-notifications.js";
import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js";
import { routineExecutionStatusService } from "../services/routine-run-status.js";
import { routineService } from "../services/routines.js";
import { issueService } from "../services/issues.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js";
import { createHeartbeatRunState } from "../services/heartbeat/run-state.js";
import { createHeartbeatRunPreparation } from "../services/heartbeat/run-preparation.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_WORK_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("routine execution status work with PostgreSQL", () => {
let db: Db, temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
let enabled = true, draining = false, standby = false;
const errors: unknown[] = [];
beforeAll(async () => {
if (!external) temporary = await startEmbeddedPostgresTestDatabase("routine-status-work-");
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
}, 60_000);
beforeEach(() => {
enabled = true; draining = false; standby = false; errors.length = 0;
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
});
afterEach(async () => {
await coordinator?.stop(); coordinator = undefined;
vi.useRealTimers(); vi.restoreAllMocks();
await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`);
});
afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); });
const canRun = () => enabled && !draining && !standby;
async function start(service = routineExecutionStatusService(db)) {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun, canReconcile: () => !standby,
onError: error => errors.push(error) });
const worker = registerRoutineExecutionStatusWork(coordinator, service, canRun);
await worker.ready;
return worker;
}
async function settled() { await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); }
async function quiet() {
await settled();
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction");
try {
await vi.advanceTimersByTimeAsync(20 * 60_000);
expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled();
} finally { select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); }
}
async function fixture() {
const companyId = randomUUID(), agentId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Routine status work", issuePrefix: `R${companyId.slice(0, 6)}`,
defaultResponsibleUserId: randomUUID(), requireBoardApprovalForNewAgents: false });
await db.insert(agents).values({ id: agentId, companyId, name: "Routine worker", status: "active", lifecycleState: "ready", adapterType: "codex_local" });
const service = routineService(db, { heartbeat: { wakeup: async () => null }, runtimeEnv: {} });
const routine = await service.create(companyId, { title: "Status projection", assigneeAgentId: agentId,
priority: "medium", status: "active", concurrencyPolicy: "always_enqueue", catchUpPolicy: "skip_missed" }, {});
return { companyId, agentId, routine, service };
}
async function dispatch(f: Awaited<ReturnType<typeof fixture>>) {
return f.service.runRoutine(f.routine.id, { source: "manual", idempotencyKey: randomUUID() });
}
async function saved(id: string) { return (await db.select().from(routineRuns).where(eq(routineRuns.id, id)))[0]; }
async function projected(id: string, status: string) {
await vi.waitFor(async () => expect((await saved(id)).status).toBe(status));
await settled();
}
it("settles empty startup, configured idle, healthy active and retained terminal history without recurring SQL", async () => {
await start(); await quiet();
const f = await fixture(); await quiet();
const run = await dispatch(f); await quiet();
await issueService(db).update(run.linkedIssueId!, { status: "done" });
await projected(run.id, "completed");
const before = await saved(run.id); await quiet();
expect(await saved(run.id)).toEqual(before); expect(errors).toEqual([]);
});
it.each(["done", "blocked", "cancelled"])("reactivates from the first ordinary %s transition and preserves input", async status => {
await start(); await quiet();
const f = await fixture(), run = await dispatch(f);
await issueService(db).update(run.linkedIssueId!, { status });
await projected(run.id, status === "done" ? "completed" : "failed");
expect((await saved(run.id)).completedAt).toBeInstanceOf(Date);
await quiet(); expect(errors).toEqual([]);
if (status !== "done") {
await issueService(db).update(run.linkedIssueId!, { status: "in_progress" });
await projected(run.id, "issue_created");
expect(await saved(run.id)).toMatchObject({ completedAt: null, failureReason: null });
await quiet();
}
});
it("reconstructs legacy terminal debt and reopened transient failures but leaves skipped and coalesced history unchanged", async () => {
const f = await fixture(), first = await dispatch(f), second = await dispatch(f);
await db.update(issues).set({ status: "done" }).where(eq(issues.id, first.linkedIssueId!));
await db.update(routineRuns).set({ status: "failed", failureReason: "Execution issue moved to blocked", completedAt: new Date() }).where(eq(routineRuns.id, second.id));
const terminal = await db.insert(routineRuns).values(["skipped", "coalesced"].map(status => ({ companyId: f.companyId,
routineId: f.routine.id, source: "manual", linkedIssueId: first.linkedIssueId, status, completedAt: new Date() }))).returning();
await start(); await projected(first.id, "completed"); await projected(second.id, "issue_created");
for (const row of terminal) expect(await saved(row.id)).toEqual(row);
await quiet(); expect(errors).toEqual([]);
});
it("settles the declared custom origin of a real plugin-managed dispatch and preserves other companies", async () => {
const f = await fixture(), other = await fixture();
const pluginKey = `routine-status-${randomUUID()}`, originId = `operation:${randomUUID()}`;
const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0",
manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test",
author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning();
await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey,
resourceKind: "routine", resourceKey: "status", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } });
const run = await dispatch(f), otherRun = await dispatch(other);
expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId);
await start(); await quiet();
await issueService(db).update(run.linkedIssueId!, { status: "done" }); await projected(run.id, "completed");
const before = await saved(run.id);
await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); });
await settled(); await quiet();
expect(await saved(run.id)).toEqual(before); expect((await saved(otherRun.id)).status).toBe("issue_created"); expect(errors).toEqual([]);
});
it.each(["changed", "removed"])("keeps historical custom-origin custody after its plugin declaration is %s", async mode => {
const f = await fixture(), pluginKey = `routine-history-${randomUUID()}`, originId = `operation:${randomUUID()}`;
const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0",
manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test",
author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning();
const [resource] = await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey,
resourceKind: "routine", resourceKey: "history", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }).returning();
const run = await dispatch(f);
expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId);
if (mode === "changed") await db.update(pluginManagedResources).set({ defaultsJson: { issueTemplate: { originId: "new-origin" } } }).where(eq(pluginManagedResources.id, resource.id));
else await db.delete(pluginManagedResources).where(eq(pluginManagedResources.id, resource.id));
await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!));
await start(); await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]);
});
it("reconstructs more than one page and does not lose a committed issue hint behind the startup cursor", async () => {
const f = await fixture(), ids = Array.from({ length: 53 }, () => randomUUID()).sort();
const issueIds = ids.map(() => randomUUID());
await db.insert(issues).values(ids.map((id, index) => ({ id: issueIds[index], companyId: f.companyId, title: "Legacy execution",
status: index === 0 ? "todo" : "done", originKind: "routine_execution", originRunId: id, originId: f.routine.id })));
await db.insert(routineRuns).values(ids.map((id, index) => ({ id, companyId: f.companyId, routineId: f.routine.id,
source: "schedule", linkedIssueId: issueIds[index], status: "issue_created" })));
const service = routineExecutionStatusService(db), load = service.load;
let entered!: () => void, release!: () => void;
const enteredGate = new Promise<void>(resolve => { entered = resolve; });
const releaseGate = new Promise<void>(resolve => { release = resolve; });
const loaded = vi.spyOn(service, "load").mockImplementationOnce(async (scope, afterId) => {
const page = await load(scope, afterId); expect(page).toHaveLength(50); entered(); await releaseGate; return page;
});
const starting = start(service); await enteredGate;
await issueService(db).update(issueIds[0]!, { status: "done" }); release(); await starting;
await vi.advanceTimersByTimeAsync(3000);
await vi.waitFor(async () => expect((await db.select().from(routineRuns)).every(row => row.status === "completed")).toBe(true));
expect(loaded.mock.calls.some(([scope, afterId]) => scope === undefined && afterId !== undefined)).toBe(true);
expect(loaded.mock.calls.some(([scope]) => scope?.issueId === issueIds[0])).toBe(true);
await quiet(); expect(errors).toEqual([]);
});
it("scopes shared execution writes to their company and settles paused-company projection debt", async () => {
const f = await fixture(), other = await fixture(), run = await dispatch(f), otherRun = await dispatch(other);
const service = routineExecutionStatusService(db), load = vi.spyOn(service, "load");
await start(service); await quiet(); load.mockClear();
await db.update(companies).set({ status: "paused" }).where(eq(companies.id, f.companyId));
await db.update(issues).set({ status: "done" }).where(eq(issues.id, otherRun.linkedIssueId!));
await writeExecutionControl(db, DELIVERY_QUEUES.executionStatus,
tx => tx.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)), { companyId: f.companyId });
await projected(run.id, "completed");
expect(load.mock.calls.every(([scope]) => scope?.companyId === f.companyId)).toBe(true);
expect((await saved(otherRun.id)).status).toBe("issue_created"); await quiet();
await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); });
await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]);
});
it("projects subtree cancel and restore without waking agents, then becomes quiet", async () => {
const f = await fixture(), root = await dispatch(f), child = await dispatch(f);
await issueService(db).update(child.linkedIssueId!, { parentId: root.linkedIssueId });
const tree = issueTreeControlService(db), actor = { actorType: "system" as const, actorId: "routine-status-test" };
await start(); await quiet();
const cancel = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "cancel", actor });
expect((await tree.cancelIssueStatusesForHold(f.companyId, root.linkedIssueId!, cancel.hold.id)).updatedIssueIds.sort())
.toEqual([root.linkedIssueId, child.linkedIssueId].sort());
await projected(root.id, "failed"); await projected(child.id, "failed"); await quiet();
const restore = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "restore", actor });
const restored = await tree.restoreIssueStatusesForHold(f.companyId, root.linkedIssueId!, restore.hold.id, { actor });
expect(restored.updatedIssues).toEqual(expect.arrayContaining([
expect.objectContaining({ id: root.linkedIssueId, status: "todo" }),
expect.objectContaining({ id: child.linkedIssueId, status: "todo" }),
]));
await projected(root.id, "issue_created"); await projected(child.id, "issue_created");
expect(await saved(root.id)).toMatchObject({ completedAt: null, failureReason: null });
expect(await saved(child.id)).toMatchObject({ completedAt: null, failureReason: null });
expect(await db.select().from(heartbeatRuns)).toEqual([]);
await quiet(); expect(errors).toEqual([]);
});
it("serializes a direct status hook with later task changes before projecting the routine", async () => {
const f = await fixture(), run = await dispatch(f); await start(); await quiet();
await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!));
let entered!: () => void, release!: () => void;
const enteredGate = new Promise<void>(resolve => { entered = resolve; });
const releaseGate = new Promise<void>(resolve => { release = resolve; });
const holding = db.transaction(async tx => {
await tx.select().from(routineRuns).where(eq(routineRuns.id, run.id)).for("update");
entered(); await releaseGate;
});
let direct: Promise<unknown> | undefined, changing: Promise<unknown> | undefined;
let directPid: number | undefined;
try {
await enteredGate; direct = f.service.syncRunStatusForIssue(run.linkedIssueId!);
await vi.waitFor(async () => {
const rows = await db.execute(sql`select pid from pg_stat_activity
where datname = current_database() and wait_event_type = 'Lock'
and (query like 'update "routine_runs"%' or query like 'select %"routine_runs"%')`);
expect(rows).toHaveLength(1); directPid = Number(rows[0]!.pid);
}, { timeout: 5000 });
await expect(db.transaction(tx => tx.select({ id: issues.id }).from(issues)
.where(eq(issues.id, run.linkedIssueId!)).for("update", { noWait: true })))
.rejects.toMatchObject({ cause: expect.objectContaining({ code: "55P03" }) });
changing = issueService(db).update(run.linkedIssueId!, { status: "blocked" });
await vi.waitFor(async () => {
const rows = await db.execute(sql`select count(*)::int as count from pg_stat_activity
where datname = current_database() and wait_event_type = 'Lock'
and ${directPid}::int = any(pg_blocking_pids(pid))`);
expect(Number(rows[0]?.count)).toBeGreaterThan(0);
}, { timeout: 5000 });
release(); await holding; await direct; await changing;
await projected(run.id, "failed"); await quiet(); expect(errors).toEqual([]);
} finally { release(); await holding; await direct; await changing; }
});
it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => {
const f = await fixture(), run = await dispatch(f); await start(); await quiet();
let release!: () => void, entered!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
const written = new Promise<void>(resolve => { entered = resolve; });
const writing = db.transaction(async tx => {
await issueService(db).update(run.linkedIssueId!, { status: "done" }, tx);
entered(); await gate;
});
try { await written; expect((await saved(run.id)).status).toBe("issue_created"); }
finally { release(); await writing; }
await projected(run.id, "completed"); await quiet();
const other = await dispatch(f);
await expect(db.transaction(async tx => {
await issueService(db).update(other.linkedIssueId!, { status: "cancelled" }, tx);
throw new Error("outer rollback");
})).rejects.toThrow("outer rollback");
await settled(); expect((await saved(other.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]);
});
it("keeps the first write during drain and reconstructs on resume without passive SQL", async () => {
const f = await fixture(), run = await dispatch(f); await start(); await quiet();
draining = true;
await issueService(db).update(run.linkedIssueId!, { status: "cancelled" });
await settled(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created");
draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "failed");
await quiet(); expect(errors).toEqual([]);
});
it.each(["standby", "disabled"])("does no owner SQL while %s and reconstructs when enabled", async mode => {
const f = await fixture(), run = await dispatch(f);
await issueService(db).update(run.linkedIssueId!, { status: "done" });
if (mode === "standby") standby = true; else enabled = false;
await start(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created");
standby = false; enabled = true; await vi.advanceTimersByTimeAsync(30_000);
await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]);
});
it("retries failed exact owners, retains the retry across drain and reconstructs after restart", async () => {
const f = await fixture(), run = await dispatch(f), service = routineExecutionStatusService(db);
const settle = vi.spyOn(service, "settle").mockRejectedValueOnce(new Error("projection unavailable"));
await start(service);
await issueService(db).update(run.linkedIssueId!, { status: "done" });
await vi.waitFor(() => expect(errors).toHaveLength(1));
expect((await saved(run.id)).status).toBe("issue_created");
await vi.advanceTimersByTimeAsync(29_000); expect(settle).toHaveBeenCalledTimes(1);
draining = true; await vi.advanceTimersByTimeAsync(60_000); await quiet();
draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "completed"); await quiet();
await coordinator!.stop(); coordinator = undefined;
const other = await dispatch(f); await issueService(db).update(other.linkedIssueId!, { status: "blocked" });
await start(); await projected(other.id, "failed"); await quiet();
});
it("preserves notifications during a sweep, coalesces duplicate hints and never overlaps", async () => {
const f = await fixture(), first = await dispatch(f), second = await dispatch(f), service = routineExecutionStatusService(db);
let entered!: () => void, release!: () => void;
const enteredGate = new Promise<void>(resolve => { entered = resolve; });
const releaseGate = new Promise<void>(resolve => { release = resolve; });
const load = service.load;
vi.spyOn(service, "load").mockImplementationOnce(async issueId => { const rows = await load(issueId); entered(); await releaseGate; return rows; });
const starting = start(service); await enteredGate;
await issueService(db).update(second.linkedIssueId!, { status: "done" });
await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); });
release(); await starting;
await projected(second.id, "completed"); expect((await saved(first.id)).status).toBe("issue_created");
await quiet(); expect(errors).toEqual([]);
});
it.each(["succeeded", "failed", "cancelled", "interrupted", "timed_out"])("uses real %s run lifecycle changes for activeIssue without an open-issue retry", async status => {
const f = await fixture(), run = await dispatch(f); await start();
const [execution] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "running", contextSnapshot: { issueId: run.linkedIssueId } }).returning();
await db.update(issues).set({ executionRunId: execution.id }).where(eq(issues.id, run.linkedIssueId!));
expect((await f.service.getDetail(f.routine.id))?.activeIssue?.id).toBe(run.linkedIssueId);
const deps = {
...createHeartbeatRunState(db), ...createHeartbeatRunPreparation(db),
issuesSvc: { addComment: vi.fn(), listReviewAttention: vi.fn() },
recovery: { escalateStrandedAssignedIssue: vi.fn() }, companySkills: { completeTestRunForIssue: vi.fn() },
budgets: { getInvocationBlock: vi.fn(async () => null) }, treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },
enqueueWakeup: vi.fn(), getCurrentUserRedactionOptions: vi.fn(async () => ({})),
getAgentInvokability: vi.fn(async () => ({ invokable: true })), emitTerminalAgentTaskRun: vi.fn(), budgetHooks: {},
} satisfies HeartbeatLifecycleDependencies;
await createHeartbeatLifecycle(db, deps).setRunStatusIfRunning(execution.id, status, { finishedAt: new Date() });
expect((await f.service.getDetail(f.routine.id))?.activeIssue).toBeNull();
expect((await saved(run.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]);
});
});
@@ -0,0 +1,107 @@
import type { RoutineExecutionStatusScope } from "../services/routine-execution-status-notifications.js";
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js";
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
let canRun = true;
const errors: unknown[] = [];
const entry = { id: "run", companyId: "company", routineId: "routine", issueId: "issue" };
beforeEach(() => {
vi.useFakeTimers(); canRun = true; errors.length = 0;
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => canRun, canReconcile: () => true,
onError: error => errors.push(error) });
});
afterEach(async () => { await coordinator.stop(); vi.useRealTimers(); });
it("loads actual debt once and leaves no recurring owner work", async () => {
const service = { load: vi.fn(async () => []), settle: vi.fn(async () => true) };
await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready;
expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
expect(coordinator.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(20 * 60_000);
expect(service.load).toHaveBeenCalledOnce(); expect(service.settle).not.toHaveBeenCalled(); expect(errors).toEqual([]);
});
it("reconstructs only an invalidated issue and coalesces duplicate requests", async () => {
const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) };
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
service.load.mockResolvedValueOnce([entry]);
worker.invalidate(JSON.stringify({ issueId: "issue" })); worker.invalidate(JSON.stringify({ issueId: "issue" }));
await vi.advanceTimersByTimeAsync(1000);
expect(service.load).toHaveBeenLastCalledWith({ issueId: "issue" }, undefined); expect(service.load).toHaveBeenCalledTimes(2);
expect(service.settle).toHaveBeenCalledExactlyOnceWith(expect.objectContaining(entry));
expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
});
it("retries only the failed owner at backoff and disarms after success", async () => {
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("locked")) };
await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready;
expect(errors).toHaveLength(1);
const retryAt = coordinator.nextWakeAt(); expect(retryAt).toBe(Date.now() + 30_000);
await vi.advanceTimersByTimeAsync(29_999); expect(service.settle).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(1); expect(service.settle).toHaveBeenCalledTimes(2);
expect(service.load).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.settle).toHaveBeenCalledTimes(2);
});
it("retains startup and first work during drain without passive owner calls", async () => {
canRun = false;
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true) };
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
worker.invalidate(JSON.stringify({ issueId: "issue" }));
await vi.advanceTimersByTimeAsync(20 * 60_000);
expect(service.load).not.toHaveBeenCalled(); expect(service.settle).not.toHaveBeenCalled();
canRun = true; await vi.advanceTimersByTimeAsync(30_000);
expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
});
it("keeps a failed owner across drain and retries after resume", async () => {
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("write failed")) };
await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready;
canRun = false; await vi.advanceTimersByTimeAsync(20 * 60_000);
expect(service.settle).toHaveBeenCalledOnce(); expect(service.load).toHaveBeenCalledOnce();
canRun = true; await vi.advanceTimersByTimeAsync(30_000);
expect(service.settle).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
});
it("retains a notification that arrives while the startup reconstruction is in flight", async () => {
let release!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) };
service.load.mockImplementationOnce(async () => { await gate; return []; });
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun);
await vi.waitFor(() => expect(service.load).toHaveBeenCalledOnce());
worker.invalidate(JSON.stringify({ issueId: "issue" })); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready;
await vi.advanceTimersByTimeAsync(1000);
expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
});
it("cancels retries and subscriptions at shutdown", async () => {
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValue(new Error("offline")) };
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
await worker.stop(); worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(20 * 60_000);
expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull();
});
it("reconstructs a backlog in bounded keyset pages while retaining a scoped hint behind the cursor", async () => {
const firstPage = Array.from({ length: 50 }, (_, i) => ({ ...entry, id: `run-${String(i).padStart(3, "0")}` }));
const late = { ...entry, id: "run-late", issueId: "late-issue" };
const service = {
load: vi.fn(async (scope?: RoutineExecutionStatusScope, afterId?: string) => scope?.issueId ? [late]
: afterId ? [{ ...entry, id: "run-050" }] : firstPage),
settle: vi.fn(async () => true),
};
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
worker.invalidate(JSON.stringify({ issueId: "late-issue" }));
await vi.advanceTimersByTimeAsync(3000);
expect(service.load).toHaveBeenCalledWith(undefined, "run-049");
expect(service.load).toHaveBeenCalledWith({ issueId: "late-issue" }, undefined);
expect(service.settle).toHaveBeenCalledTimes(52);
expect(service.settle).toHaveBeenCalledWith(expect.objectContaining(late));
expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
});
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -5,11 +6,12 @@ import { companies, createDb, issues, routineRuns, routines } from "@paperclipai
import { routineService } from "../services/routines.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_TEST_DATABASE_URL");
const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
if (!support.supported) console.warn(`Skipping routine status database tests: ${support.reason}`);
(support.supported ? describe : describe.skip)("routine status reconciliation", () => {
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(database.connectionString); }, 60_000);
beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(external ?? database.connectionString, { maxConnections: 6 }); }, 60_000);
afterEach(async () => { vi.restoreAllMocks(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`); });
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
async function seed(status = "blocked") {
@@ -36,11 +38,10 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${
expect((await db.select().from(routineRuns))[0]).toMatchObject({ status: "issue_created", completedAt: null });
}
});
it.each(["active_issue", "wrong_origin", "wrong_routine", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => {
it.each(["active_issue", "wrong_origin", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => {
const f = await seed();
if (mode === "active_issue") await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id));
if (mode === "wrong_origin") await db.update(issues).set({ originKind: "manual" }).where(eq(issues.id, f.issue.id));
if (mode === "wrong_routine") await db.update(issues).set({ originId: randomUUID() }).where(eq(issues.id, f.issue.id));
if (mode === "wrong_run") await db.update(issues).set({ originRunId: randomUUID() }).where(eq(issues.id, f.issue.id));
if (mode === "wrong_company") { const other = await seed("done"); await db.update(routineRuns).set({ companyId: other.companyId }).where(eq(routineRuns.id, f.run.id)); }
if (mode === "wrong_routine_company") { const other = await seed("done");
@@ -53,6 +54,20 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${
expect((await db.select().from(routineRuns)).find(row => row.id === f.run.id)).toEqual(before.find(row => row.id === f.run.id));
if (mode === "wrong_routine_company") await expect(routineService(db).syncRunStatusForIssue(f.issue.id)).resolves.toBeNull();
});
it("uses reciprocal durable custody when a retained origin no longer names the current routine declaration", async () => {
const f = await seed("done");
await db.update(issues).set({ originId: "previous-declaration-origin" }).where(eq(issues.id, f.issue.id));
expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 1, settled: 1, deferred: 0 });
expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("completed");
});
it.each(["unknown-status", "other-code"])("preserves a failed invocation with an unrelated transient marker: %s", async mode => {
const f = await seed("in_progress");
await db.update(routineRuns).set({ status: "failed", failureReason: null,
triggerPayload: { transientFailure: { code: mode === "other-code" ? "unrelated_failure" : "execution_issue_status",
status: mode === "unknown-status" ? "unrecognized" : "blocked" } } }).where(eq(routineRuns.id, f.run.id));
expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 0, settled: 0, deferred: 0 });
expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("failed");
});
it.each(["resume", "retarget", "routine_transfer"])("rechecks %s after candidate selection and before repair", async mode => {
const f = await seed(); const transaction = db.transaction.bind(db);
vi.spyOn(db, "transaction").mockImplementationOnce(async callback => {
@@ -1,3 +1,6 @@
import { routineExecutionStatusService } from "../services/routine-run-status.js";
import { attentionRetentionService } from "../services/attention-retention.js";
import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js";
import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js";
import { skillVersionRetentionService } from "../services/skill-version-retention.js";
import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js";
@@ -134,6 +137,9 @@ const {
})),
reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })),
reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })),
registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
registerRetainedNativeSessionCleanupWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve(), stop: vi.fn() })),
@@ -180,6 +186,8 @@ const {
const heartbeatServiceFactoryMock = vi.fn(() => heartbeatServiceMock);
const issueThreadInteractionServiceMock = {
sweepSupersededPendingRequestConfirmations: vi.fn(async () => ({ expired: 0 })),
mergedPullRequestConfirmationPage: vi.fn(async () => ({ rows: [], complete: true })),
mergedPullRequestConfirmationCandidates: vi.fn(async () => []),
sweepMergedPullRequestConfirmations: vi.fn(async () => ({
checked: 0,
candidates: 0,
@@ -378,6 +386,11 @@ vi.mock("../realtime/live-events-ws.js", () => ({
setupLiveEventsWebSocketServer: vi.fn(),
}));
vi.mock("../services/attention-retention.js", async importOriginal => ({
...await importOriginal<typeof import("../services/attention-retention.js")>(),
attentionRetentionService: vi.fn(() => ({ load: vi.fn(async () => ({ rows: [], complete: true })), check: vi.fn() })),
}));
vi.mock("../services/join-request-retention.js", async importOriginal => ({
...await importOriginal<typeof import("../services/join-request-retention.js")>(),
joinRequestRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })),
@@ -393,6 +406,11 @@ vi.mock("../services/approval-retention.js", async importOriginal => ({
approvalRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })),
}));
vi.mock("../services/routine-run-status.js", async importOriginal => ({
...await importOriginal<typeof import("../services/routine-run-status.js")>(),
routineExecutionStatusService: vi.fn(() => ({ load: vi.fn(async () => []), settle: vi.fn(async () => false) })),
}));
vi.mock("../services/index.js", () => ({
backfillLegacyToolOAuthTokens: vi.fn(async () => ({
scannedConnections: 0,
@@ -528,6 +546,13 @@ vi.mock("../services/native-runtime/native-question-bridge.js", () => ({
validateNativeQuestionResponseInput: vi.fn(),
}));
vi.mock("../services/status-card-evaluation.js", () => ({
statusCardEvaluationService: vi.fn(() => ({ run: vi.fn(async () => {}), next: vi.fn(async () => null), changed: vi.fn() })),
registerStatusCardEvaluationWork: vi.fn((_coordinator, service, canRun) => ({
ready: canRun() ? service.run(new AbortController().signal) : Promise.resolve(), wake: vi.fn(), stop: vi.fn(async () => {}),
})),
}));
vi.mock("../services/secret-proposals.js", () => ({
createSecretProposalsService: vi.fn(() => ({
expiryDeadlines: vi.fn(async () => []),
@@ -791,6 +816,28 @@ describe("startServer feedback export wiring", () => {
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.skillVersionRetention)).toEqual([]);
});
it.each([false, true])("registers status-card evaluation only with maintenance and never ticks it from the heartbeat loop (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 1000 }));
let tick: (() => void) | undefined;
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => {
if (interval === 1000) tick = callback;
return 1 as unknown as ReturnType<typeof setInterval>;
}) as typeof setInterval);
try {
await startServer(); await settleStartupHousekeeping();
expect(registerStatusCardEvaluationWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
expect(statusCardEvaluationService).toHaveBeenCalledTimes(enabled ? 1 : 0);
if (enabled) {
const service = vi.mocked(statusCardEvaluationService).mock.results.at(-1)!.value;
expect(service.run).toHaveBeenCalledOnce();
expect(tick).toBeDefined();
for (let i = 0; i < 3; i++) { tick!(); await settleStartupHousekeeping(); }
expect(service.run).toHaveBeenCalledOnce();
}
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.statusCardEvaluation)).toEqual([]);
} finally { timer.mockRestore(); }
});
it.each([false, true])("registers response recovery and its shutdown subscription only when maintenance is enabled (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
await startServer(); await settleStartupHousekeeping();
@@ -924,6 +971,21 @@ describe("startServer feedback export wiring", () => {
if (enabled) expect(heartbeatServiceMock.registerCapturedInstructionCommitWork.mock.calls[0][1]()).toBe(true);
});
it.each([false, true])("registers retry and queued dispatch only when maintenance is enabled (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
await startServer(); await settleStartupHousekeeping();
expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled();
if (enabled) {
const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0];
const app = await createAppMock.mock.results.at(-1)!.value;
expect(coordinator).toBe(app.locals.deliveryWork);
expect(admitted()).toBe(true);
}
});
it.each([false, true])("registers retry cancellation recovery only when maintenance is enabled (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
await startServer(); await settleStartupHousekeeping();
@@ -975,7 +1037,7 @@ describe("startServer feedback export wiring", () => {
}
});
it.each([false, true])("registers archive delivery separately from retention sweeps (scheduler %s)", async enabled => {
it.each([false, true])("registers source retention separately from archive delivery without tick scans (scheduler %s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30000 }));
let tick: (() => void) | undefined;
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => {
@@ -987,12 +1049,17 @@ describe("startServer feedback export wiring", () => {
const registration = deliveryWorkRegisterMock.mock.calls.find(([queue]) => queue === DELIVERY_QUEUES.decisionArchiveDelivery);
if (!enabled) {
expect(registration).toBeUndefined();
expect(attentionRetentionService).not.toHaveBeenCalled();
expect(decisionRetentionService).not.toHaveBeenCalled();
expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping();
expect(decisionRetentionService).not.toHaveBeenCalled();
return;
}
expect(registration).toBeDefined();
expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.attentionRetention)).toBe(true);
expect(attentionRetentionService).toHaveBeenCalledOnce();
const source = vi.mocked(attentionRetentionService).mock.results[0]!.value;
expect(source.load).toHaveBeenCalledOnce();
const retention = vi.mocked(decisionRetentionService).mock.results[0]!.value;
expect(retention.deliverNotifications).toHaveBeenCalledOnce();
expect(retention.nextArchiveNotificationAt).toHaveBeenCalledOnce();
@@ -1000,9 +1067,10 @@ describe("startServer feedback export wiring", () => {
const companyReads = () => vi.mocked(db.select).mock.results.filter(({ value }) =>
vi.mocked(value.from).mock.calls.some(([table]) => table === companies)).length;
const before = companyReads();
expect(before).toBeGreaterThan(0); expect(tick).toBeDefined();
expect(before).toBe(0); expect(tick).toBeDefined();
tick!(); await settleStartupHousekeeping();
expect(companyReads()).toBeGreaterThan(before);
expect(companyReads()).toBe(before);
expect(source.load).toHaveBeenCalledOnce();
expect(retention.deliverNotifications).toHaveBeenCalledOnce();
await registration![1].run(new AbortController().signal);
expect(retention.deliverNotifications).toHaveBeenCalledTimes(2);
@@ -1322,13 +1390,14 @@ describe("startServer feedback export wiring", () => {
await startServer();
await settleStartupHousekeeping();
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1);
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
expect(intervalCallback).not.toBeNull();
intervalCallback?.();
await new Promise<void>((resolve) => setImmediate(resolve));
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2);
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled();
expect(retiredDetector).not.toHaveBeenCalled();
} finally {
delete (runtime as Partial<typeof runtime>).reconcileProductivityReviews;
@@ -1367,6 +1436,27 @@ describe("startServer feedback export wiring", () => {
}
});
it.each([{ cloud: false, enabled: true }, { cloud: true, enabled: false }, { cloud: true, enabled: true }])(
"registers routine status only for enabled Cloud scheduling ($cloud/$enabled)", async ({ cloud, enabled }) => {
if (cloud) vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
else vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
const intervalCallbacks: Array<() => void> = [];
const intervals = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void) => {
intervalCallbacks.push(callback); return 1 as unknown as ReturnType<typeof setInterval>;
}) as typeof setInterval);
try {
await startServer(); await settleStartupHousekeeping();
expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.routineExecutionStatus)).toHaveLength(cloud && enabled ? 1 : 0);
const service = vi.mocked(routineExecutionStatusService).mock.results.at(-1)?.value;
if (cloud && enabled) expect(service?.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
for (let index = 0; index < 4; index++) for (const callback of intervalCallbacks) callback();
await settleStartupHousekeeping();
if (cloud && enabled) expect(service?.load).toHaveBeenCalledOnce();
} finally { intervals.mockRestore(); vi.unstubAllEnvs(); }
},
);
it("keeps routine ticks and setup cleanup active when heartbeat scheduling is suppressed", async () => {
loadConfigMock.mockReturnValue(buildTestConfig({
heartbeatSchedulerEnabled: true,
@@ -1403,7 +1493,9 @@ describe("startServer feedback export wiring", () => {
expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1);
expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function));
expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true);
expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1);
expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalledTimes(1);
expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).not.toHaveBeenCalled();
expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.mergedPullRequestConfirmations)).toBe(true);
expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1);
expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1);
const routineWorkerIndex = deliveryWorkRegisterMock.mock.calls.findIndex(([queue]) => queue === DELIVERY_QUEUES.routineSchedules);
@@ -1501,6 +1593,20 @@ describe("startServer feedback export wiring", () => {
}
});
it("becomes ready without waiting for the merge-confirmation observation", async () => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true }));
let release!: () => void;
const blocked = new Promise<{ rows: never[]; complete: boolean }>(resolve => { release = () => resolve({ rows: [], complete: true }); });
issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage.mockReturnValueOnce(blocked);
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => 1 as unknown as ReturnType<typeof setInterval>);
let started = false;
const starting = startServer().then(server => { started = true; return server; });
try {
await vi.waitFor(() => expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalled());
await settleStartupHousekeeping(); expect(started).toBe(true); expect(getStartupRecoveryState().phase).toBe("ready");
} finally { release(); await starting; timer.mockRestore(); }
});
it.each(["retained cleanup", "workspace finalization", "environment settlement", "native run projection", "native result commit", "native recovery notices", "external conversation settlement"] as const)("becomes ready while %s is held but joins it at shutdown", async owner => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true }));
let releaseCleanup!: () => void;
@@ -0,0 +1,656 @@
import { budgetService } from "../services/budgets.js";
import { documentService } from "../services/documents.js";
import * as companySearch from "../services/company-search.js";
import { setImmediate as immediate } from "node:timers/promises";
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, withDedicatedDbConnection,
heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db";
import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "../services/status-card-evaluation.js";
import { statusCardService } from "../services/status-cards.js";
import { issueService } from "../services/issues.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARD_EVALUATION_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("status-card evaluation work", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
let draining = false;
let companyId: string, agentId: string;
const errors: unknown[] = [];
const evaluations = new Set<Promise<unknown>>();
let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] });
const actor = { agentId: null, userId: "board-user" };
const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({
mode, debounceSeconds: 60, intervalMinutes: 15,
});
// Controlled provider boundary: assignment persists a queued wake, and tests
// explicitly run the authorized summarizer writeback. No model/provider runs.
const wakeup = vi.fn(async (id: string, options: { payload?: Record<string, unknown> | null; reason?: string | null }) => {
const [agent] = await db.select({ companyId: agents.companyId }).from(agents).where(eq(agents.id, id));
return db.insert(agentWakeupRequests).values({ companyId: agent!.companyId, agentId: id, source: "assignment",
reason: options.reason, payload: options.payload, status: "queued" }).returning();
});
beforeAll(async () => {
if (!external) temporary = await startEmbeddedPostgresTestDatabase("status-card-evaluation-");
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
}, 30_000);
beforeEach(async () => {
companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockReset();
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] });
await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
await instanceSettingsService(db).updateExperimental({ enableStatusCards: true });
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
});
afterEach(async () => {
await coordinator?.stop(); coordinator = undefined;
vi.useRealTimers(); vi.restoreAllMocks();
await db.delete(budgetIncidents); await db.delete(budgetPolicies); await db.delete(approvals); await db.delete(costEvents);
await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards);
await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog);
await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns);
await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
async function start(service = statusCardEvaluationService(db, { wakeup })) {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
onError: error => errors.push(error) });
await registerStatusCardEvaluationWork(coordinator, { ...service, run: signal => {
const work = service.run(signal);
evaluations.add(work);
void work.then(() => evaluations.delete(work), () => evaluations.delete(work));
return work;
} }, () => !draining).ready;
}
async function settled() {
// Do not leap across PostgreSQL connect timers while real I/O is pending.
// Await actual domain work, then require several settled event-loop turns
// so a transient zero hold cannot stand in for transaction reconciliation.
for (let turn = 0; turn < 3; turn++) {
await vi.waitFor(async () => {
await Promise.allSettled([...evaluations]);
await vi.advanceTimersByTimeAsync(500);
await Promise.allSettled([...evaluations]);
expect(idleWorkSnapshot().active).toBe(0);
}, { timeout: 5000 });
await immediate();
}
}
async function quiet() {
await settled();
expect(coordinator!.nextWakeAt()).toBeNull();
const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute");
const calls = wakeup.mock.calls.length;
await vi.advanceTimersByTimeAsync(10 * 60_000);
expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled();
expect(wakeup).toHaveBeenCalledTimes(calls);
reads.mockRestore(); writes.mockRestore(); executes.mockRestore();
}
async function writeback(cardId: string, generationIssueId: string, compile = false) {
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId));
const writer = { agentId, runId: run!.id };
const cards = statusCardService(db);
if (compile) await cards.writeQuery(cardId, { queries: [compiledQuery],
title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer);
await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer);
await issueService(db).update(generationIssueId, { status: "done" });
}
async function card(mode: "manual" | "reactive" | "interval" = "reactive", refreshPolicy = policy(mode)) {
const cards = statusCardService(db);
const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy }, actor);
const compile = await cards.requestCompile(created.id, actor);
await writeback(created.id, compile.generatingIssue.id, true);
return cards.getById(created.id).then(row => row!);
}
it("settles empty, disabled, configured manual and unchanged reactive history without recurring SQL", async () => {
await start(); await quiet();
await card("manual"); await card("reactive"); await quiet();
await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet();
expect(errors).toEqual([]);
});
it("ordinary issue change schedules debounce, assigns one durable wake, writes a result and becomes quiet", async () => {
await start(); const created = await card(); await quiet();
const changed = await issueService(db).create(companyId, { title: "New blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
await vi.waitFor(async () => {
expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0);
}, { timeout: 5000 });
const pending = await statusCardService(db).getById(created.id);
expect(pending?.pendingChangeCount).toBeGreaterThan(0);
expect(pending?.nextEvalAt).toBeInstanceOf(Date);
expect(wakeup).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(60_000); await settled();
const generating = await statusCardService(db).getById(created.id);
expect(generating?.generatingIssueId).toBeTruthy();
const durable = await db.select().from(agentWakeupRequests);
expect(durable).toHaveLength(1); expect(durable[0]?.payload?.issueId).toBe(generating?.generatingIssueId);
expect(changed.companyId).toBe(companyId);
await writeback(created.id, generating!.generatingIssueId!); await quiet();
expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === generating?.generatingIssueId)))
.toMatchObject([{ status: "ok", finishedAt: expect.any(Date) }]);
expect(errors).toEqual([]);
});
it("keeps card deletion scoped to its company and a missing deletion quiet", async () => {
const removed = await card();
await card();
const ownCompany = companyId, ownAgent = agentId;
companyId = randomUUID(); agentId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
const other = await card();
companyId = ownCompany; agentId = ownAgent;
const factory = companySearch.companySearchService;
const searchedCompanies: string[] = [];
vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => {
const search = factory(owner), actual = search.search;
search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); };
return search;
});
await start(); await quiet(); searchedCompanies.length = 0;
const cards = statusCardService(db);
expect(await cards.remove(removed.id)).toMatchObject({ id: removed.id, companyId: ownCompany });
await settled();
// Deleting a card only reconstructs deadlines; it is not an issue change
// that warrants re-running any unchanged reactive card's saved search.
expect(searchedCompanies).toEqual([]);
expect(await cards.getById(other.id)).toMatchObject({ nextEvalAt: null, generatingIssueId: null, pendingChangeCount: 0 });
expect(wakeup).not.toHaveBeenCalled();
searchedCompanies.length = 0;
expect(await cards.remove(removed.id)).toBeNull();
await quiet();
expect(searchedCompanies).toEqual([]);
expect(errors).toEqual([]);
});
it("coalesces one hundred committed source writes without delaying another company's saved deadline", async () => {
const reactive = await card();
const originalCompany = companyId, originalAgent = agentId;
companyId = randomUUID(); agentId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
const other = await card("interval");
const otherIssue = await issueService(db).create(companyId, { title: "Other interval blocker", status: "blocked", priority: "medium" });
const otherReactive = await card();
companyId = originalCompany; agentId = originalAgent;
const service = statusCardEvaluationService(db, { wakeup });
const originalRun = service.run;
let running: Promise<unknown> | undefined;
const run = vi.spyOn(service, "run").mockImplementation(signal => {
const result = originalRun(signal); running = result; return result;
});
await start(service); await settled();
await vi.advanceTimersByTimeAsync(1000);
// Give each committed producer and any actual domain work time to settle.
// This prevents incidental overlap from hiding per-write reconstruction.
const flush = async () => {
for (let i = 0; i < 10; i++) {
await running; await immediate();
}
};
await flush();
const select = vi.spyOn(db, "select");
const factory = companySearch.companySearchService;
const searchedCompanies: string[] = [];
vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => {
const search = factory(owner);
const actual = search.search;
search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); };
return search;
});
run.mockClear();
const burstStarted = Date.now();
for (let i = 0; i < 50; i++) {
const issue = await issueService(db).create(companyId, { title: `Burst blocker ${i}`, status: "blocked", priority: "medium" });
await flush();
await issueService(db).addComment(issue.id, `Human update ${i}`, { userId: "board-user" });
await flush();
}
await flush();
// One eager reconciliation after idle and one bounded batched continuation;
// the producers themselves are not counted as background SQL.
const dueReads = () => select.mock.calls.filter(([fields]) => fields && "card" in fields && fields.card === statusCards).length;
const nextReads = () => select.mock.calls.filter(([fields]) => fields && "at" in fields && fields.at === statusCards.nextEvalAt).length;
const elapsedMs = Date.now() - burstStarted;
const maximum = Math.ceil(elapsedMs / 1000) + 2;
console.info("status-card burst", { elapsedMs, maximum, runs: run.mock.calls.length, due: dueReads(), next: nextReads(), searches: searchedCompanies.length });
expect(run.mock.calls.length).toBeLessThanOrEqual(maximum);
expect(dueReads()).toBeLessThanOrEqual(maximum); expect(nextReads()).toBeLessThanOrEqual(maximum);
expect(searchedCompanies.length).toBeLessThanOrEqual(maximum);
expect(searchedCompanies.length).toBeGreaterThan(0);
expect(new Set(searchedCompanies)).toEqual(new Set([originalCompany]));
expect(wakeup).not.toHaveBeenCalled();
// Re-register with an existing durable deadline inside the coalescing
// window. A company hint must not make that real deadline wait one second.
await coordinator!.stop(); coordinator = undefined;
const dueAt = Date.now() + 500;
await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id));
await start(service); await flush();
await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" });
await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id,
"More changes while another company is due", { userId: "board-user" });
await flush();
await vi.advanceTimersByTimeAsync(500); await flush();
await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce());
expect(Date.now()).toBeLessThan(dueAt + 500);
expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent);
const generated = (await statusCardService(db).getById(other.id))!;
expect(generated.generatingIssueId).toBeTruthy();
expect((await statusCardService(db).getById(reactive.id))!.pendingChangeCount).toBeGreaterThan(0);
expect((await statusCardService(db).getById(otherReactive.id))!.pendingChangeCount).toBeGreaterThan(0);
expect(searchedCompanies).toContain(other.companyId);
// Complete all ordinary results, then prove there is no recurring batch
// timer once the requested work has settled.
companyId = other.companyId; agentId = other.agentId!;
await writeback(other.id, generated.generatingIssueId!);
await statusCardService(db).update((await statusCardService(db).getById(other.id))!, { archived: true }, actor);
companyId = originalCompany; agentId = originalAgent;
await vi.advanceTimersByTimeAsync(120_000); await settled(); await flush();
await vi.waitFor(async () => expect((await statusCardService(db).getById(reactive.id))!.generatingIssueId).toBeTruthy());
const pending = (await statusCardService(db).getById(reactive.id))!;
expect(pending.generatingIssueId).toBeTruthy();
await writeback(reactive.id, pending.generatingIssueId!);
const otherPending = (await statusCardService(db).getById(otherReactive.id))!;
expect(otherPending.generatingIssueId).toBeTruthy();
companyId = other.companyId; agentId = other.agentId!;
await writeback(otherReactive.id, otherPending.generatingIssueId!);
companyId = originalCompany; agentId = originalAgent;
select.mockRestore();
await quiet(); expect(errors).toEqual([]);
}, 30_000);
it("bounds active-hours conversion work across twenty cards and repeated ordinary company writes", async () => {
vi.setSystemTime(new Date("2026-11-02T18:00:00Z"));
const hoursPolicy = { ...policy(), activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } };
const cards = [];
for (let i = 0; i < 20; i++) cards.push(await card("reactive", hoursPolicy));
await start(); await quiet();
const RealFormatter = Intl.DateTimeFormat;
const actualParts = RealFormatter.prototype.formatToParts;
let constructions = 0, conversions = 0;
const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
if (args[0] === "en-GB" && args[1]?.hourCycle === "h23") constructions++;
return new RealFormatter(...args);
} as unknown as typeof Intl.DateTimeFormat);
const parts = vi.spyOn(RealFormatter.prototype, "formatToParts").mockImplementation(function (this: Intl.DateTimeFormat, date) {
if (this.resolvedOptions().locale === "en-GB" && this.resolvedOptions().hourCycle === "h23") conversions++;
return actualParts.call(this, date);
});
const target = await issueService(db).create(companyId, { title: "Closed-hours blocker", status: "blocked", priority: "medium" });
await settled();
expect(constructions).toBeLessThanOrEqual(1);
expect(conversions).toBeLessThanOrEqual(1000);
const cold = { constructions, conversions };
constructions = 0; conversions = 0;
for (let i = 0; i < 10; i++) {
await issueService(db).addComment(target.id, `Closed-hours human update ${i}`, { userId: "board-user" });
await settled();
}
expect(constructions).toBe(0);
expect(conversions).toBeGreaterThanOrEqual(200);
expect(conversions).toBeLessThanOrEqual(800);
expect(wakeup).not.toHaveBeenCalled();
const opening = new Date("2026-11-03T09:00:00Z");
for (const created of cards) expect((await statusCardService(db).getById(created.id)))
.toMatchObject({ state: "paused_hours", pendingChangeCount: 1, nextEvalAt: opening });
process.stdout.write(JSON.stringify({ measurement: "status-card company active-hours workload", cards: 20,
sourceWrites: 11, cold, warm: { constructions, conversions } }) + "\n");
constructor.mockRestore(); parts.mockRestore();
await vi.advanceTimersByTimeAsync(opening.getTime() - Date.now()); await settled();
expect(wakeup).toHaveBeenCalledTimes(20);
for (const created of cards) {
const generated = (await statusCardService(db).getById(created.id))!;
expect(generated.generatingIssueId).toBeTruthy();
await writeback(created.id, generated.generatingIssueId!);
}
await quiet(); expect(errors).toEqual([]);
}, 60_000);
it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => {
await start(); let created = await card("interval"); await settled();
const old = coordinator!.nextWakeAt();
created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor);
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(100);
await Promise.allSettled([...evaluations]);
expect(coordinator!.nextWakeAt()).toBeLessThan(old!);
});
await statusCardService(db).update(created, { archived: true }, actor); await quiet();
expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]);
});
it("restores overdue debt after the feature is disabled across its saved deadline", async () => {
await start(); const created = await card(); await quiet();
await issueService(db).create(companyId, { title: "Disabled blocker", status: "blocked", priority: "medium" });
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
const pending = (await statusCardService(db).getById(created.id))!;
await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet();
expect(wakeup).not.toHaveBeenCalled();
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt);
await instanceSettingsService(db).updateExperimental({ enableStatusCards: true });
await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce());
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); await quiet();
});
it("does not react to rolled-back or nested uncommitted card configuration", async () => {
await start(); const created = await card("manual"); await quiet();
await expect(db.transaction(async tx => {
await statusCardService(tx as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor);
await vi.advanceTimersByTimeAsync(30_000); expect(wakeup).not.toHaveBeenCalled();
throw new Error("rollback");
})).rejects.toThrow("rollback");
await quiet(); expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull();
await db.transaction(async tx => {
await tx.transaction(async nested => {
await statusCardService(nested as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor);
});
expect(coordinator!.nextWakeAt()).not.toBeNull();
expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull();
});
await settled(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]);
});
it("serializes competing connections when one source event preempts a future evaluation", async () => {
const created = await card();
await db.update(statusCards).set({ nextEvalAt: new Date(Date.now() + 3_600_000) }).where(eq(statusCards.id, created.id));
await issueService(db).create(companyId, { title: "Concurrent blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
const first = statusCardEvaluationService(db, { wakeup });
const second = statusCardEvaluationService(db, { wakeup });
first.changed(new Set([`changes:${companyId}`])); second.changed(new Set([`changes:${companyId}`]));
await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]);
const pending = await statusCardService(db).getById(created.id);
expect(pending?.pendingChangeCount).toBeGreaterThan(0);
expect(pending?.nextEvalAt?.getTime()).toBeLessThan(Date.now() + 3_600_000);
expect(wakeup).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(60_000);
await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]);
const updated = await statusCardService(db).getById(created.id);
expect(updated?.generatingIssueId).toBeTruthy(); expect(wakeup).toHaveBeenCalledOnce();
expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === updated?.generatingIssueId)))
.toHaveLength(1);
});
it("finishes generation and writeback with a one-connection pool and releases dedicated locks", async () => {
vi.useRealTimers();
const previousDb = db;
db = createDb(external ?? temporary!.connectionString, { maxConnections: 1, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
try {
const created = await card("reactive", { ...policy(), debounceSeconds: 1 });
const countLocks = async () => Number((await db.execute<{ count: string }>(sql`
select count(*) from pg_stat_activity where datname = current_database()
and application_name = 'paperclip-workspace-finalization-lock'`))[0]!.count);
const baseline = await countLocks();
await issueService(db).create(companyId, { title: "Pool-one blocker", status: "blocked", priority: "medium" });
const service = statusCardEvaluationService(db, { wakeup });
const acquired = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
const holder = withDedicatedDbConnection(db, connection => connection.transaction(async tx => {
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`);
acquired.resolve(); await release.promise;
}));
await acquired.promise;
try {
await service.run(new AbortController().signal);
expect(wakeup).not.toHaveBeenCalled();
expect(await service.next()).toBeGreaterThan(Date.now());
} finally { release.resolve(); await holder; }
await service.run(new AbortController().signal);
expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0);
await vi.waitFor(async () => {
await service.run(new AbortController().signal);
expect(wakeup).toHaveBeenCalledOnce();
}, { timeout: 5000, interval: 100 });
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!);
await service.run(new AbortController().signal);
expect(await service.next()).toBeNull();
expect(idleWorkSnapshot().active).toBe(0);
await vi.waitFor(async () => expect(await countLocks()).toBe(baseline));
// Failure exits also release the physical connection and advisory lock.
await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id));
await service.run(new AbortController().signal);
expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("retry");
await vi.waitFor(async () => expect(await countLocks()).toBe(baseline));
await withDedicatedDbConnection(db, connection => connection.transaction(async tx => {
expect((await tx.execute<{ acquired: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${created.id}, 0)) as acquired`))[0]!.acquired).toBe(true);
}));
} finally { await db.$client.end({ timeout: 1 }); db = previousDb; }
}, 15_000);
it.each([false, true])("recovers a lost committed hint before a future membership deadline (legacy=%s)", async legacy => {
await issueService(db).create(companyId, { title: "Original blocker", status: "blocked", priority: "medium" });
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1d" });
const created = await card();
expect(created.nextEvalReason).toBe("membership");
const oldExpiry = created.nextEvalAt!.getTime();
// No coordinator exists to receive this committed source hint. A fresh
// service has no in-memory hint and must reconstruct from durable truth.
await issueService(db).create(companyId, { title: "Committed before crash", status: "blocked", priority: "medium" });
if (legacy) await db.update(statusCards).set({ nextEvalReason: null }).where(eq(statusCards.id, created.id));
const restarted = statusCardEvaluationService(db, { wakeup });
await restarted.run(new AbortController().signal);
const pending = (await statusCardService(db).getById(created.id))!;
expect(pending.pendingChangeCount).toBe(1);
expect(pending.nextEvalReason).toBe("policy");
expect(pending.nextEvalAt!.getTime()).toBeLessThan(oldExpiry);
expect(wakeup).not.toHaveBeenCalled();
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
expect(wakeup).toHaveBeenCalledOnce();
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!);
expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("membership");
});
it("preserves a legacy ambiguous five-minute claim without extending it on restart", async () => {
const created = await card();
const retry = new Date(Date.now() + 240_000);
await db.update(statusCards).set({ nextEvalAt: retry, nextEvalReason: null }).where(eq(statusCards.id, created.id));
await issueService(db).create(companyId, { title: "Unconsumed source", status: "blocked", priority: "medium" });
const restarted = statusCardEvaluationService(db, { wakeup });
await restarted.run(new AbortController().signal);
expect(await restarted.next()).toBe(retry.getTime());
vi.setSystemTime(retry); await restarted.run(new AbortController().signal);
expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0);
});
it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => {
vi.setSystemTime(new Date("2026-11-02T18:00:00Z"));
const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}),
...(gate === "rate" ? { maxUpdatesPerHour: 1 } : {}), ...(gate === "budget" ? { dailyTokenCap: 100 } : {}) };
const created = await card("reactive", refreshPolicy);
if (gate === "rate" || gate === "budget") await db.insert(statusCardUpdates).values({ cardId: created.id, kind: "full", trigger: "reactive",
status: "ok", startedAt: new Date(Date.now() - (gate === "rate" ? 30 * 60_000 : 60_000)), finishedAt: new Date(), inputTokens: gate === "budget" ? 100 : 0 });
await issueService(db).create(companyId, { title: "Saved blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
const service = statusCardEvaluationService(db, { wakeup });
await service.run(new AbortController().signal);
const pending = (await statusCardService(db).getById(created.id))!;
expect(pending.pendingChangeCount).toBeGreaterThan(0); expect(pending.nextEvalAt!.getTime()).toBeGreaterThan(Date.now());
const restarted = statusCardEvaluationService(db, { wakeup });
await restarted.run(new AbortController().signal);
expect(await restarted.next()).toBe(pending.nextEvalAt!.getTime());
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt);
vi.setSystemTime(pending.nextEvalAt!.getTime() - 1); await restarted.run(new AbortController().signal);
expect(wakeup).not.toHaveBeenCalled();
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
expect(wakeup).toHaveBeenCalledOnce();
});
it("unions startup reconstruction with a scoped source change in another company", async () => {
const firstCompany = companyId;
const first = await card();
await issueService(db).create(companyId, { title: "First blocker", status: "blocked", priority: "medium" });
companyId = randomUUID(); agentId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `X${companyId.slice(0, 6)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
const second = await card();
await issueService(db).create(companyId, { title: "Second blocker", status: "blocked", priority: "medium" });
const service = statusCardEvaluationService(db, { wakeup });
service.changed(new Set([`changes:${companyId}`]));
await service.run(new AbortController().signal);
const cards = statusCardService(db);
expect((await cards.getById(first.id))?.companyId).toBe(firstCompany);
expect((await cards.getById(first.id))?.pendingChangeCount).toBeGreaterThan(0);
expect((await cards.getById(second.id))?.pendingChangeCount).toBeGreaterThan(0);
await vi.advanceTimersByTimeAsync(60_000); await service.run(new AbortController().signal);
expect(wakeup).toHaveBeenCalledTimes(2);
expect(new Set(await db.select().from(agentWakeupRequests).then(rows => rows.map(row => row.companyId))))
.toEqual(new Set([firstCompany, companyId]));
});
it("keeps ignored issue activity quiet and reacts to human comments and document search membership", async () => {
await start(); await card(); await quiet();
await issueService(db).create(companyId, { title: "Ignored todo", status: "todo", priority: "medium" });
await quiet(); expect(wakeup).not.toHaveBeenCalled();
compiledQuery = companySearchQuerySchema.parse({ scope: "all", q: "launch" });
const searchCard = await card(); await quiet();
const target = await issueService(db).create(companyId, { title: "Ordinary task", status: "todo", priority: "medium" });
await quiet();
await documentService(db).upsertIssueDocument({ issueId: target.id, key: "plan", format: "markdown", body: "launch evidence", createdByUserId: "board-user" });
await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0));
await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce();
await writeback(searchCard.id, (await statusCardService(db).getById(searchCard.id))!.generatingIssueId!); await quiet();
await issueService(db).addComment(target.id, "A human launch update", { userId: "board-user" });
await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0));
});
it("retains relative query membership expiry across restart and emits the eventual removal", async () => {
const now = new Date();
const target = await issueService(db).create(companyId, { title: "Expiring blocker", status: "blocked", priority: "medium" });
await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, target.id));
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h" });
const created = await card();
const expires = now.getTime() + 3_600_001;
expect(created.nextEvalAt?.getTime()).toBe(expires);
const restarted = statusCardEvaluationService(db, { wakeup });
await restarted.run(new AbortController().signal);
expect(await restarted.next()).toBe(expires);
vi.setSystemTime(expires - 1); await restarted.run(new AbortController().signal); expect(wakeup).not.toHaveBeenCalled();
vi.setSystemTime(expires); await restarted.run(new AbortController().signal);
const pending = (await statusCardService(db).getById(created.id))!;
expect(pending.pendingChangeCount).toBe(1); expect(pending.nextEvalAt!.getTime()).toBe(expires + 60_000);
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
expect(wakeup).toHaveBeenCalledOnce();
});
it("uses expiry of an off-page candidate to refresh offset-ranked query membership", async () => {
const now = new Date();
const tasks = await Promise.all((["critical", "high", "medium"] as const).map(priority => issueService(db).create(companyId, {
title: `${priority} blocker`, priority, status: "blocked",
})));
await db.update(issues).set({ updatedAt: new Date(now.getTime() - 59 * 60_000) }).where(eq(issues.id, tasks[0]!.id));
for (const task of tasks.slice(1)) await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, task.id));
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h", sort: "priority", limit: 1, offset: 1 });
const created = await card();
expect(Object.keys(created.fingerprint!)).toEqual([tasks[1]!.id]);
const expires = now.getTime() + 60_001;
expect(created.nextEvalAt?.getTime()).toBe(expires);
const service = statusCardEvaluationService(db, { wakeup });
vi.setSystemTime(expires); await service.run(new AbortController().signal);
const pending = (await statusCardService(db).getById(created.id))!;
expect(pending.pendingChangeCount).toBe(2); expect(wakeup).not.toHaveBeenCalled();
vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal);
expect(wakeup).toHaveBeenCalledOnce();
const [update] = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.generationIssueId, (await statusCardService(db).getById(created.id))!.generatingIssueId!));
expect(new Set(update!.changes.map(change => change.issueId))).toEqual(new Set([tasks[1]!.id, tasks[2]!.id]));
});
it("retains a scoped retry while another evaluator owns the card lock", async () => {
const created = await card();
await issueService(db).create(companyId, { title: "Locked blocker", status: "blocked", priority: "medium" });
const acquired = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
const holder = db.transaction(async tx => {
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`);
acquired.resolve(); await release.promise;
});
await acquired.promise;
const service = statusCardEvaluationService(db, { wakeup });
service.changed(new Set([`changes:${companyId}`]));
try {
await service.run(new AbortController().signal);
expect(await service.next()).toBeGreaterThan(Date.now());
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull();
expect(wakeup).not.toHaveBeenCalled();
} finally { release.resolve(); await holder; }
await vi.advanceTimersByTimeAsync(30_000); await service.run(new AbortController().signal);
const pending = (await statusCardService(db).getById(created.id))!;
expect(pending.pendingChangeCount).toBeGreaterThan(0);
vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal);
expect(wakeup).toHaveBeenCalledOnce();
});
it("retains the durable five-minute retry after evaluation failure across restart and drain", async () => {
const created = await card();
await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id));
const service = statusCardEvaluationService(db, { wakeup });
await service.run(new AbortController().signal);
const failed = (await statusCardService(db).getById(created.id))!;
const retryAt = failed.nextEvalAt!.getTime();
expect(retryAt).toBeGreaterThan(Date.now() + 299_000); expect(wakeup).not.toHaveBeenCalled();
await start(); expect(coordinator!.nextWakeAt()).toBe(retryAt);
draining = true; await vi.advanceTimersByTimeAsync(5 * 60_000); await settled(); expect(wakeup).not.toHaveBeenCalled();
// Repair the controlled malformed stored-query fault; no tested result is precreated.
await db.update(statusCards).set({ queries: [compiledQuery] }).where(eq(statusCards.id, created.id));
draining = false; await vi.advanceTimersByTimeAsync(30_000); await quiet();
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull();
});
it("cancels failed assignments without rerunning a completed generation and settles after retry", async () => {
await start(); const created = await card(); await quiet();
wakeup.mockRejectedValueOnce(new Error("assignment unavailable"));
await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" });
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
await vi.advanceTimersByTimeAsync(60_000); await settled();
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect((await statusCardService(db).getById(created.id))!.state).toBe("error");
});
const failed = (await statusCardService(db).getById(created.id))!;
expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null, nextEvalReason: null });
expect(errors.length).toBeGreaterThan(0);
const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id));
const ledger = history.find(row => row.status === "failed")!;
expect(ledger.finishedAt).toBeInstanceOf(Date);
expect((await db.select().from(issues).where(eq(issues.id, ledger.generationIssueId!)))[0]?.status).toBe("cancelled");
const count = wakeup.mock.calls.length; await quiet(); expect(wakeup).toHaveBeenCalledTimes(count);
// An explicit ordinary refresh is still the user's supported retry action.
const retry = await statusCardService(db).requestRefresh(created.id, { trigger: "manual", actor });
expect(retry.generatingIssue).toBeTruthy(); await writeback(created.id, retry.generatingIssue!.id); await quiet();
expect((await statusCardService(db).getById(created.id))!.state).toBe("active");
});
it("budget-owned company resume reactivates reactive work committed while paused", async () => {
await start(); const created = await card(); await quiet();
const budgets = budgetService(db);
await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 100, hardStopEnabled: true, notifyEnabled: false }, "board-user");
const [cost] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", billingType: "metered_api", model: "test",
costCents: 200, occurredAt: new Date() }).returning();
await budgets.evaluateCostEvent(cost!);
expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("paused");
await quiet();
await issueService(db).create(companyId, { title: "Paused blocker", status: "blocked", priority: "medium" }); await quiet();
expect(wakeup).not.toHaveBeenCalled();
await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 1000 }, "board-user");
expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("active");
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce();
});
it("preserves the first write during drain until resume", async () => {
await start(); const created = await card("manual"); await quiet();
draining = true;
await statusCardService(db).update(created, { refreshPolicy: policy("interval") }, actor);
await settled(); expect(wakeup).not.toHaveBeenCalled();
draining = false; await vi.advanceTimersByTimeAsync(30_000); await settled();
expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]);
});
});
@@ -0,0 +1,31 @@
import { expect, it, vi } from "vitest";
import { installDatabaseWorkSignals, subscribeDatabaseWork } from "../../../packages/db/src/work-signals.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { notifyExecutionRunWork, writeExecutionControl } from "../services/execution-control-notifications.js";
it.each([false, true])("keeps execution queue consumers intact and emits only a known status-card company scope (%s)", async scoped => {
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: async (fn: (tx: object) => Promise<unknown>) =>
fn({ execute: async () => [{ xid: "1" }] }) });
const queues = [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement,
DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors];
const settled: Array<{ queue: string; hints?: string[] }> = [];
const unsubscribe = [...queues, DELIVERY_QUEUES.statusCardEvaluation, DELIVERY_QUEUES.workspaceRecovery].map(queue => subscribeDatabaseWork(owner, queue, (event, hints) => {
if (event === "settled") settled.push({ queue, hints: hints ? [...hints] : undefined });
}));
try {
const scope = scoped ? { companyId: "changed-company", runId: "changed-run" } : undefined;
await owner.transaction(async tx => {
await notifyExecutionRunWork(tx, scope);
expect(settled).toEqual([]);
});
expect(new Set(settled.map(row => row.queue))).toEqual(new Set([...queues, ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])]));
expect(settled.find(row => row.queue === DELIVERY_QUEUES.statusCardEvaluation)?.hints)
.toEqual(scoped ? ["changes:changed-company"] : undefined);
settled.length = 0;
const write = vi.fn(async () => "written");
expect(await writeExecutionControl(owner as never, DELIVERY_QUEUES.workspaceRecovery, write, scope)).toBe("written");
expect(write).toHaveBeenCalledOnce();
expect(new Set(settled.map(row => row.queue))).toEqual(new Set([DELIVERY_QUEUES.issueMonitors, DELIVERY_QUEUES.workspaceRecovery,
...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])]));
} finally { for (const stop of unsubscribe) stop(); }
});
@@ -1,4 +1,4 @@
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { statusCardRefreshPolicySchema } from "@paperclipai/shared";
import {
chooseStatusCardUpdateKind,
@@ -8,10 +8,12 @@ import {
filterStatusCardChanges,
isWithinStatusCardActiveHours,
nextStatusCardEvaluationAt,
nextStatusCardPolicyEligibility,
statusCardChangesHash,
} from "../services/status-card-update-engine.js";
describe("status card update engine", () => {
afterEach(() => vi.restoreAllMocks());
const defaultPolicy = statusCardRefreshPolicySchema.parse({ mode: "interval", intervalMinutes: 15 });
it("retains non-terminal and terminal status transitions plus membership changes", () => {
@@ -78,6 +80,149 @@ describe("status card update engine", () => {
expect(evaluateStatusCardPolicy({ policy: hours, now: new Date("2026-07-23T18:00:00.000Z"), lastChangeAt: null, updatesLastHour: 0, tokensToday: 0, manual: false }).action).toBe("pause_hours");
});
it("disarms unchanged reactive evaluation while retaining exact pending eligibility", () => {
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 90, maxUpdatesPerHour: 2,
activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } });
const now = new Date("2026-07-23T16:59:30Z");
expect(nextStatusCardEvaluationAt(policy, now)).toBeNull();
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 }))
.toEqual(new Date("2026-07-24T09:00:00Z"));
const open = new Date("2026-07-23T14:00:00Z");
expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: new Date("2026-07-23T13:00:00Z"),
completedUpdateTimes: [new Date("2026-07-23T13:30:00Z"), new Date("2026-07-23T13:45:00Z")], tokensToday: 0 }))
.toEqual(new Date("2026-07-23T14:30:00.001Z"));
expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: open, completedUpdateTimes: [], tokensToday: 100_000 }))
.toEqual(new Date("2026-07-24T09:00:00Z"));
});
it("retains empty-window pending changes without a recurring eligibility scan", () => {
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
activeHours: { start: "09:00", end: "09:00", timezone: "UTC" } });
const now = new Date("2026-07-23T14:00:00Z");
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull();
});
it("bounds empty-window formatter work before any timezone conversion", () => {
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
activeHours: { start: "09:00", end: "09:00", timezone: "Pacific/Chatham" } });
const formatter = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function () {
throw new Error("Empty active hours must not construct a formatter");
});
const now = new Date("2026-07-23T14:00:00Z");
expect(isWithinStatusCardActiveHours(policy, now)).toBe(false);
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull();
expect(formatter).not.toHaveBeenCalled();
});
it("bounds formatter construction and reuses the next opening across cards and source writes", () => {
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
activeHours: { start: "09:00", end: "17:00", timezone: "America/St_Johns" } });
const RealFormatter = Intl.DateTimeFormat;
let constructions = 0;
vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
if (++constructions > 1) throw new Error("Card evaluation repeatedly constructed timezone formatters");
return new RealFormatter(...args);
} as unknown as typeof Intl.DateTimeFormat);
const conversions = vi.spyOn(RealFormatter.prototype, "formatToParts");
const now = new Date("2026-07-23T22:00:00Z");
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
const started = performance.now();
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("2026-07-24T11:30:00Z"));
const coldMs = performance.now() - started;
const coldConversions = conversions.mock.calls.length;
expect(coldConversions).toBeLessThanOrEqual(2881);
conversions.mockClear();
const warmStarted = performance.now();
for (let write = 0; write < 100; write++) for (let card = 0; card < 20; card++) {
expect(nextStatusCardPolicyEligibility({ ...input,
policy: { ...policy, activeHours: { ...policy.activeHours! } },
now: new Date(now.getTime() + write * 1000),
})).toEqual(new Date("2026-07-24T11:30:00Z"));
}
expect(constructions).toBeLessThanOrEqual(1);
expect(conversions.mock.calls.length).toBeLessThanOrEqual(2000);
process.stdout.write(JSON.stringify({ measurement: "status-card active-hours cost", coldMs, coldConversions, constructions,
warmMs: performance.now() - warmStarted, warmConversions: conversions.mock.calls.length, evaluations: 2000 }) + "\n");
});
it.each([
["America/New_York", "02:15", "03:30", "2026-03-08T06:50:00Z", "2026-03-08T07:00:00Z"],
["America/New_York", "02:15", "02:45", "2026-03-08T06:50:00Z", "2026-03-09T06:15:00Z"],
["America/New_York", "02:15", "02:45", "2026-03-07T07:45:00Z", "2026-03-09T06:15:00Z"],
["America/New_York", "01:15", "01:45", "2026-11-01T05:50:00Z", "2026-11-01T06:15:00Z"],
["America/New_York", "23:30", "01:30", "2026-11-01T05:40:00Z", "2026-11-01T06:00:00Z"],
["Australia/Lord_Howe", "02:15", "02:45", "2026-10-03T15:20:00Z", "2026-10-03T15:30:00Z"],
["Australia/Lord_Howe", "01:35", "01:45", "2026-04-04T14:50:00Z", "2026-04-04T15:05:00Z"],
["Asia/Kathmandu", "09:00", "17:00", "2026-07-23T12:00:00Z", "2026-07-24T03:15:00Z"],
["Pacific/Apia", "00:10", "00:20", "2011-12-30T09:50:00Z", "2011-12-30T10:10:00Z"],
])("preserves the first real opening in %s %s-%s after %s", (timezone, start, end, time, expected) => {
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, activeHours: { start, end, timezone } });
const now = new Date(time);
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date(expected));
// A later closed instant reuses the opening, while the exact opening itself
// remains eligible without rounding it into the next minute or day.
expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(now.getTime() + 1000) })).toEqual(new Date(expected));
const opening = new Date(expected);
expect(nextStatusCardPolicyEligibility({ ...input, now: opening })).toEqual(opening);
expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(opening.getTime() + 1) })).toEqual(new Date(opening.getTime() + 1));
});
it("invalidates an upcoming opening for clock rollback and changed policy values", () => {
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } });
const calculate = (time: string, activeHours = policy.activeHours!) => {
const now = new Date(time);
return nextStatusCardPolicyEligibility({ policy: { ...policy, activeHours }, now,
lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 });
};
expect(calculate("2026-07-23T18:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z"));
expect(calculate("2026-07-23T08:00:00Z")).toEqual(new Date("2026-07-23T09:00:00Z"));
expect(calculate("2026-07-23T17:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z"));
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, start: "10:00" })).toEqual(new Date("2026-07-24T10:00:00Z"));
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, end: "19:00" })).toEqual(new Date("2026-07-23T18:00:00Z"));
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, timezone: "Asia/Tokyo" })).toEqual(new Date("2026-07-24T00:00:00Z"));
});
it("checks the precise instant before a cached minute opening for historical second offsets", () => {
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
activeHours: { start: "09:00", end: "10:00", timezone: "Africa/Monrovia" } });
const now = new Date("1970-01-01T08:00:00Z");
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("1970-01-01T09:45:00Z"));
const inside = new Date("1970-01-01T09:44:45Z");
expect(isWithinStatusCardActiveHours(policy, inside)).toBe(true);
expect(nextStatusCardPolicyEligibility({ ...input, now: inside })).toEqual(inside);
});
it("bounds formatter and upcoming-opening cache retention", () => {
const now = new Date("2026-07-25T00:00:00Z");
const timezonePolicy = (timezone: string) => statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
activeHours: { start: "09:00", end: "17:00", timezone } });
const zones = Intl.supportedValuesOf("timeZone").slice(0, 34);
const first = timezonePolicy(zones[0]!);
isWithinStatusCardActiveHours(first, now);
for (const zone of zones.slice(1)) isWithinStatusCardActiveHours(timezonePolicy(zone), now);
const RealFormatter = Intl.DateTimeFormat;
const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
return new RealFormatter(...args);
} as unknown as typeof Intl.DateTimeFormat);
isWithinStatusCardActiveHours(first, now);
expect(constructor).toHaveBeenCalledTimes(1);
constructor.mockRestore();
const time = (minute: number) => `${String(Math.floor(minute / 60)).padStart(2, "0")}:${String(minute % 60).padStart(2, "0")}`;
const calculate = (minute: number) => {
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
activeHours: { start: time(minute), end: time(minute + 1), timezone: "UTC" } });
return nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 });
};
calculate(1);
for (let minute = 2; minute <= 258; minute++) calculate(minute);
const conversions = vi.spyOn(Intl.DateTimeFormat.prototype, "formatToParts");
expect(calculate(1)).toEqual(new Date("2026-07-25T00:01:00Z"));
expect(conversions.mock.calls.length).toBeGreaterThan(1);
});
it("selects full rebuilds for bounded drift rules and incremental otherwise", () => {
const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false };
expect(chooseStatusCardUpdateKind(base)).toBe("incremental");
+8 -3
View File
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
@@ -37,7 +38,8 @@ import {
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARDS_TEST_DATABASE_URL");
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
type Db = ReturnType<typeof createDb>;
@@ -79,8 +81,8 @@ describeEmbeddedPostgres("status card routes", () => {
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-");
db = createDb(tempDb.connectionString);
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-");
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
}, 20_000);
afterEach(async () => {
@@ -100,6 +102,7 @@ describeEmbeddedPostgres("status card routes", () => {
});
afterAll(async () => {
await db?.$client.end({ timeout: 1 });
await tempDb?.cleanup();
});
@@ -189,6 +192,7 @@ describeEmbeddedPostgres("status card routes", () => {
.send({ title: "Launch health", titlePinned: true });
expect(patched.status).toBe(200);
expect(patched.body).toMatchObject({ title: "Launch health", titlePinned: true });
expect(patched.body).not.toHaveProperty("nextEvalReason");
const scheduled = await request(app)
.patch(`/api/status-cards/${created.body.id}`)
@@ -1149,6 +1153,7 @@ describeEmbeddedPostgres("status card routes", () => {
});
expect(summaryWrite.status).toBe(200);
expect(summaryWrite.body.card).toMatchObject({ state: "active", queryVersion: 1, generatingIssueId: null });
expect(summaryWrite.body.card).not.toHaveProperty("nextEvalReason");
expect(summaryWrite.body.document.latestBody).toContain("**Decide:**");
expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "full"))).toMatchObject({ status: "ok", finishedAt: expect.any(Date), inputTokens: 5200, outputTokens: 980 });
+26 -2
View File
@@ -1,3 +1,5 @@
import { PgDialect } from "drizzle-orm/pg-core";
import type { SQL } from "drizzle-orm";
import { describe, expect, it, vi } from "vitest";
import {
enrichWorkProductMetadataWithDiff,
@@ -33,6 +35,18 @@ function createWorkProductRow(overrides: Partial<Record<string, unknown>> = {})
};
}
function retentionReads(insideTransaction: () => boolean) {
return vi.fn(async (query: SQL) => {
const statement = new PgDialect().sqlToQuery(query);
expect(insideTransaction()).toBe(true);
expect(statement.params).toContain("company-1");
expect(statement.params).toContain("issue-1");
if (statement.sql.includes("origin_kind")) return [{ id: "issue-1", originKind: null, originId: null }];
if (statement.sql.includes("with recursive affected")) return [{ issueId: "issue-1" }];
throw new Error(`Unexpected work-product fixture query: ${statement.sql}`);
});
}
describe("workProductService", () => {
it("extracts runner totals and enriches work-product metadata", () => {
const summary = workProductDiffSummaryFromEventPayload({
@@ -158,11 +172,16 @@ describe("workProductService", () => {
const insertValues = vi.fn(() => ({ returning: insertReturning }));
const txInsert = vi.fn(() => ({ values: insertValues }));
let transactionOpen = false;
const tx = {
execute: retentionReads(() => transactionOpen),
update: txUpdate,
insert: txInsert,
};
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => await callback(tx));
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => {
transactionOpen = true;
try { return await callback(tx); } finally { transactionOpen = false; }
});
const svc = workProductService({ transaction } as any);
const result = await svc.createForIssue("issue-1", "company-1", {
@@ -194,11 +213,16 @@ describe("workProductService", () => {
const updateSet = vi.fn(() => ({ where: updateWhere }));
const txUpdate = vi.fn(() => ({ set: updateSet }));
let transactionOpen = false;
const tx = {
execute: retentionReads(() => transactionOpen),
select: txSelect,
update: txUpdate,
};
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => await callback(tx));
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => {
transactionOpen = true;
try { return await callback(tx); } finally { transactionOpen = false; }
});
const svc = workProductService({ transaction } as any);
const result = await svc.update("work-product-1", {
+25 -159
View File
@@ -1,3 +1,7 @@
import { registerMergeConfirmationWork } from "./services/merged-pr-confirmation-work.js";
import { registerRoutineExecutionStatusWork } from "./services/routine-execution-status-work.js";
import { routineExecutionStatusService } from "./services/routine-run-status.js";
import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js";
import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js";
import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js";
import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js";
@@ -109,7 +113,6 @@ import { createApp } from "./app.js";
import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js";
import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js";
import { logger } from "./middleware/logger.js";
import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js";
import { setStartupRecoveryPhase } from "./startup-recovery-state.js";
import {
StartupRefusalError,
@@ -127,7 +130,6 @@ import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./mi
import {
feedbackService,
applyManagedEnvironments,
attentionService,
backfillPrincipalAccessCompatibility,
backfillLegacyToolOAuthTokens,
bootstrapExecutionPolicyFromEnv,
@@ -145,13 +147,12 @@ import {
reconcileCodexLocalManagedHomesOnStartup,
reconcilePersistedRuntimeServicesOnStartup,
routineService,
statusCardService,
toolAccessService,
workspaceOperationService,
} from "./services/index.js";
import { questionResponseDeliveryService } from "./services/question-response-delivery.js";
import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js";
import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js";
import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "./services/status-card-evaluation.js";
import { createSecretProposalsService } from "./services/secret-proposals.js";
import { environmentRuntimeService } from "./services/environment-runtime.js";
import { historicalCleanupVerificationService } from "./services/historical-cleanup-verification.js";
@@ -973,9 +974,6 @@ async function startServerWithDatabaseTeardown(
}
};
const pluginWorkerManager = createPluginWorkerManager();
const reportIncompleteWatchdogScans = createIncompleteTaskWatchdogScanReporter(
(details, message) => logger.warn(details, message),
);
const heartbeat = config.heartbeatSchedulerEnabled
? heartbeatService(db as any, { pluginWorkerManager })
: null;
@@ -1554,16 +1552,7 @@ async function startServerWithDatabaseTeardown(
heartbeat.drainActiveRunExecutions();
prepareHotRestartShutdown = heartbeat.prepareHotRestartShutdown;
const routines = routineService(db as any, { pluginWorkerManager });
let nextRoutineStatusReconcile = 0;
const scheduleRoutineStatusReconcile = () => {
if (!isCloudManagedInstance() || isWarmStandby() || isIdleTaskDrainActive() || heartbeatSchedulerStopped ||
Date.now() < nextRoutineStatusReconcile) return;
nextRoutineStatusReconcile = Date.now() + 5 * 60_000;
trackHeartbeatSchedulerWork(routines.reconcileExecutionRunStatuses().then(result => {
if (result.settled > 0) logger.info(result, "reconciled routine execution issue status");
}).catch(() => logger.warn("routine execution status reconciliation deferred")));
};
const statusCards = statusCardService(db as any);
const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat);
const issues = issueService(db as any);
const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, {
wakeup: heartbeat.wakeup,
@@ -1594,19 +1583,8 @@ async function startServerWithDatabaseTeardown(
: config.workspaceCompactionDelayHours * 60 * 60 * 1000,
terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS,
});
const scheduleMergedPullRequestConfirmationSweep = () => {
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
trackHeartbeatSchedulerWork(mergedPullRequestConfirmations
.sweepMergedPullRequestConfirmations()
.then((result) => {
if (result.accepted > 0) {
logger.info(result, "accepted merge confirmations for merged pull requests");
}
})
.catch((err) => {
logger.error({ err }, "merged pull-request confirmation sweep failed");
}));
};
registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations,
() => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby());
const gitRestorePointWorker = registerGitRestorePointWork(deliveryWork, gitRestorePointRetentionService(db as any, {
storage: storageService,
retentionDays: config.gitRestorePointRetentionDays,
@@ -1767,8 +1745,6 @@ async function startServerWithDatabaseTeardown(
}
}
const promotion = await timeStartupRecovery("scheduled_retries", () => heartbeat.promoteDueScheduledRetries());
await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns());
const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions());
if (
recoveredGoalActions.enqueued > 0 ||
@@ -1786,48 +1762,10 @@ async function startServerWithDatabaseTeardown(
"startup session-goal recovery resumed durable agent goals",
);
}
const reconciled = await heartbeat.reconcileStrandedAssignedIssues();
if (
promotion.promoted > 0 ||
reconciled.assignmentDispatched > 0 ||
reconciled.dispatchRequeued > 0 ||
reconciled.continuationRequeued > 0 ||
reconciled.successfulRunHandoffEscalated > 0 ||
reconciled.successfulRunHandoffRetried > 0 ||
reconciled.escalated > 0
) {
logger.warn(
{ promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled },
"startup heartbeat recovery changed assigned issue state",
);
}
const dependencyWakesReconciled = await heartbeat.reconcileResolvedDependencyWakes();
if (dependencyWakesReconciled.healed > 0) {
logger.warn(
{ ...dependencyWakesReconciled },
"startup dependency-wake reconciliation restored task execution paths",
);
}
const taskWatchdogsReconciled = await heartbeat.reconcileTaskWatchdogs();
reportIncompleteWatchdogScans(taskWatchdogsReconciled, "startup");
if (taskWatchdogsReconciled.triggered > 0) {
logger.warn(
{ ...taskWatchdogsReconciled },
"startup task-watchdog reconciliation triggered watchdog work",
);
}
const scanned = await heartbeat.scanSilentActiveRuns();
if (scanned.created > 0 || scanned.escalated > 0) {
logger.warn({ ...scanned }, "startup active-run output watchdog created review work");
}
const swept = await heartbeat.sweepStaleIssueLocks();
if (swept.cleared > 0) {
logger.warn({ ...swept }, "startup stale-lock sweeper cleared issue locks");
}
})().catch((err) => {
logger.error({ err }, "startup heartbeat recovery failed");
throw err;
@@ -1836,6 +1774,12 @@ async function startServerWithDatabaseTeardown(
await startupHeartbeatRecovery;
}
if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork,
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork,
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork,
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork,
() => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready;
@@ -1915,23 +1859,6 @@ async function startServerWithDatabaseTeardown(
if (config.heartbeatSchedulerEnabled) await heartbeat.registerStoppedInstructionCollectionWork(deliveryWork,
() => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready;
const runRetentionSweep = async () => {
const activeCompanies = await db.select({ id: companies.id }).from(companies).where(eq(companies.status, "active"));
let archived = 0;
for (const company of activeCompanies) {
// Cursor pagination rebuilds the whole feed for every page; one
// unscoped all-items build keeps this sweep at a single feed build
// per company per tick.
const page = await attentionService(db as any).list(company.id, {
includeDismissed: true,
excludeSources: ["approval", "join_request"],
all: true,
allowUnscopedAll: true,
});
archived += await retentionExecutor.autoArchive({ companyId: company.id, items: page.items });
}
return { archived };
};
// Startup housekeeping: provider and connection sweeps that every timer tick
// repeats and that neither serving requests nor run recovery depends on.
// Readiness does not wait for it. It is tracked as scheduler work, so idle
@@ -1974,7 +1901,10 @@ async function startServerWithDatabaseTeardown(
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any),
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
await step("retention_sweep", runRetentionSweep);
await step("status_card_evaluation", () => registerStatusCardEvaluationWork(deliveryWork, statusCardEvaluation,
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
await step("attention_retention", () => registerAttentionRetentionWork(deliveryWork, attentionRetentionService(db as any),
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
};
const createHeartbeatSchedulerTick = (scheduled: ReturnType<typeof registerScheduledWork>) => () => {
@@ -1983,10 +1913,6 @@ async function startServerWithDatabaseTeardown(
// captures the authoritative set of running heartbeat rows.
trackHeartbeatSchedulerWork((async () => {
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
scheduleRoutineStatusReconcile();
trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => {
logger.error({ err }, "decision retention sweep failed");
}));
const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses();
if (sweptRuntimeStatuses > 0) {
logger.info(
@@ -1998,89 +1924,24 @@ async function startServerWithDatabaseTeardown(
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
scheduleMergedPullRequestConfirmationSweep();
scheduleGitHubConnectionEventPoll();
scheduleGitHubConnectionContinuitySweep();
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
trackHeartbeatSchedulerWork((async () => {
const experimental = await instanceSettingsService(db).getExperimental();
if (experimental.enableStatusCards !== true) return;
const result = await statusCards.tickDueStatusCards(new Date());
await Promise.all(result.enqueued.map(async ({ cardId, generatingIssue }) => {
try {
await queueIssueAssignmentWakeup({
heartbeat,
issue: generatingIssue,
reason: "status_card_update_assigned",
mutation: "status_card.scheduler_update_requested",
contextSource: "status_card_scheduler",
requestedByActorType: "system",
taskKey: `status-card:${cardId}`,
rethrowOnError: true,
});
} catch (err) {
await issues.update(generatingIssue.id, { status: "cancelled" });
throw err;
}
}));
if (result.evaluated > 0 || result.enqueued.length > 0) {
logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card scheduler tick complete");
}
})().catch((err) => {
logger.error({ err }, "status-card scheduler tick failed");
}));
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) {
// Periodically reap orphaned runs (5-min staleness threshold) and make sure
// persisted queued work is still being driven forward.
runPeriodicRecovery(() => heartbeat
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
.then(() => heartbeat.promoteDueScheduledRetries())
.then(async (promotion) => {
await heartbeat.resumeQueuedRuns();
const reconciled = await heartbeat.reconcileStrandedAssignedIssues({ skipExternalConversationSettlement: true });
if (
promotion.promoted > 0 ||
reconciled.assignmentDispatched > 0 ||
reconciled.dispatchRequeued > 0 ||
reconciled.continuationRequeued > 0 ||
reconciled.successfulRunHandoffEscalated > 0 ||
reconciled.successfulRunHandoffRetried > 0 ||
reconciled.escalated > 0
) {
logger.warn(
{ promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled },
"periodic heartbeat recovery changed assigned issue state",
);
}
})
.then(async () => {
const reconciled = await heartbeat.reconcileResolvedDependencyWakes();
if (reconciled.healed > 0) {
logger.warn({ ...reconciled }, "periodic dependency-wake reconciliation restored task execution paths");
}
})
.then(async () => {
const reconciled = await heartbeat.reconcileTaskWatchdogs();
reportIncompleteWatchdogScans(reconciled, "periodic");
if (reconciled.triggered > 0) {
logger.warn({ ...reconciled }, "periodic task-watchdog reconciliation triggered watchdog work");
}
})
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
.then(async () => {
const scanned = await heartbeat.scanSilentActiveRuns();
if (scanned.created > 0 || scanned.escalated > 0) {
logger.warn({ ...scanned }, "periodic active-run output watchdog created review work");
}
})
.then(async () => {
const swept = await heartbeat.sweepStaleIssueLocks();
if (swept.cleared > 0) {
logger.warn({ ...swept }, "periodic stale-lock sweeper cleared issue locks");
}
})
.catch((err) => {
logger.error({ err }, "periodic heartbeat recovery failed");
}));
@@ -2090,6 +1951,11 @@ async function startServerWithDatabaseTeardown(
}));
};
const startHeartbeatScheduler = () => {
if (isCloudManagedInstance()) {
const status = registerRoutineExecutionStatusWork(deliveryWork, routineExecutionStatusService(db as any),
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped);
trackHeartbeatSchedulerWork(status.ready);
}
const scheduled = registerScheduledWork(deliveryWork, {
retryMs: config.heartbeatSchedulerIntervalMs,
tickAgentTimers: async signal => {
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js";
import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
@@ -385,6 +386,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
));
}
await notifyStatusCardIssueChange(tx, companyId);
await tx
.update(issues)
.set({
@@ -40,7 +40,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
let companyId: string;
const workers: ReturnType<typeof configureAgentLifecycle>[] = [];
beforeAll(async () => {
if (external) { await applyPendingMigrations(external); db = createDb(external); }
if (external) { await applyPendingMigrations(external); db = createDb(external, { maxConnections: 6 }); }
else { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); }
}, 90_000);
afterAll(async () => {
@@ -80,22 +80,20 @@ export function agentRecords(db: Db, effects: LifecycleEffects) {
throw conflict("Only agents in error status can have their error cleared");
}
const updated = await db
.update(agents)
.set({
status: "idle",
pauseReason: null,
pausedAt: null,
errorReason: null,
updatedAt: new Date(),
})
.where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready")))
.returning()
.then((rows) => rows[0] ?? null);
const updated = await effects.transaction(db, existing.companyId, async tx => {
const [current] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState })
.from(agents).where(and(eq(agents.id, id), eq(agents.companyId, existing.companyId))).for("no key update");
if (current?.status !== "error" || current.lifecycleState !== "ready") {
throw conflict("Only agents in error status can have their error cleared");
}
await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle");
const [saved] = await tx.update(agents).set({
status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(),
}).where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))).returning();
if (!saved) throw conflict("Only agents in error status can have their error cleared");
return saved;
});
if (!updated) {
throw conflict("Only agents in error status can have their error cleared");
}
return getById(updated.id);
},
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js";
import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js";
import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js";
@@ -511,6 +512,15 @@ export function createPostgresRunDispatchAdapter(
return decideScheduledRetryGate(factsResult.facts, input.now);
}
async function nextScheduledRetryAt(input: { cutoff: Date | null }): Promise<number | null> {
const [row] = await db.select({ at: heartbeatRuns.scheduledRetryAt }).from(heartbeatRuns)
.where(and(eq(heartbeatRuns.status, "scheduled_retry"),
sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`,
input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined))
.orderBy(asc(heartbeatRuns.scheduledRetryAt)).limit(1);
return row?.at?.getTime() ?? null;
}
async function listDueRetries(input: ListDueRetriesInput): Promise<DueRetryRun[]> {
const rows = await db
.select()
@@ -518,6 +528,7 @@ export function createPostgresRunDispatchAdapter(
.where(
and(
eq(heartbeatRuns.status, "scheduled_retry"),
sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`,
lte(heartbeatRuns.scheduledRetryAt, input.now),
input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined,
),
@@ -986,6 +997,7 @@ export function createPostgresRunDispatchAdapter(
recordedCommentId && authors.get(recordedCommentId) === actorId ? [recordedCommentId] : [];
}));
if (commentIds.some(id => !recordedCommentIds.has(id))) return null;
await notifyDeferredExecutionWork(tx, { companyId: run.companyId, requestId: wake.id });
const [deferred] = await tx.update(agentWakeupRequests).set({
status: "deferred_issue_execution", runId: null, claimedAt: null, finishedAt: null, error: null,
payload: withQueuedCommentIdsInWakePayload({ ...payload, _paperclipWakeContext: context,
@@ -1414,6 +1426,7 @@ export function createPostgresRunDispatchAdapter(
return {
evaluateScheduledRetryGate,
nextScheduledRetryAt,
listDueRetries,
cancelStaleQueuedRun,
dispatchResolvedInteractionIfCurrent,
@@ -24,6 +24,7 @@ export type EvaluateScheduledRetryGateInput = {
/** Read-only scheduled-retry operations exposed to application use cases. */
export interface ScheduledRetryReader {
nextScheduledRetryAt(input: { cutoff: Date | null }): Promise<number | null>;
evaluateScheduledRetryGate(input: EvaluateScheduledRetryGateInput): Promise<GateDecision>;
listDueRetries(input: ListDueRetriesInput): Promise<DueRetryRun[]>;
}
@@ -12,6 +12,7 @@ function fakeReader(dueRuns: DueRetryRun[] = []): ScheduledRetryReader & { evalu
const evaluateCalls: unknown[] = [];
return {
evaluateCalls,
async nextScheduledRetryAt() { return null; },
async evaluateScheduledRetryGate(input) {
evaluateCalls.push(input);
return { allowed: true };
+1
View File
@@ -67,6 +67,7 @@ export function createRunDispatch(db: Db, deps: RunDispatchDeps = {}) {
});
return {
nextScheduledRetryAt: (input: { cutoff: Date | null }) => adapter.nextScheduledRetryAt(input),
evaluateScheduledRetryGate: createEvaluateScheduledRetryGate({ reader: adapter }),
promoteScheduledRetry,
promoteDueScheduledRetries: createPromoteDueScheduledRetries({
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js";
import { instanceSettingsService } from "../../../services/instance-settings.js";
import { randomBytes, randomUUID } from "node:crypto";
@@ -37,7 +38,8 @@ import type { TransactionScope } from "../application/ports.js";
// a foreign-company row is invisible to a read, and a deferred-status
// compare-and-set that affects no row leaves no other trace. The decision
// branching itself is proven against plain facts in `domain/policy.test.ts`.
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WAKE_QUEUE_TEST_DATABASE_URL");
const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
if (!embeddedPostgresSupport.supported) {
@@ -57,8 +59,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
};
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
db = createDb(tempDb.connectionString);
if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 });
}, 20_000);
afterEach(async () => {
@@ -78,6 +80,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
});
afterAll(async () => {
await db?.$client.end({ timeout: 1 });
await tempDb?.cleanup();
});
@@ -1,3 +1,6 @@
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js";
import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js";
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
@@ -610,6 +613,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
},
async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
await notifyStatusCardIssueChange(tx, companyId);
const executionState = parseIssueExecutionState(issue.executionState);
const wakeupRequest = await tx
.insert(agentWakeupRequests)
@@ -637,6 +641,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
.returning()
.then((rows) => rows[0]);
await notifyRunDispatch(tx, { companyId });
const queuedRun = await tx
.insert(heartbeatRuns)
.values({
@@ -715,6 +720,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
.returning()
.then((rows) => rows[0]);
await notifyRunDispatch(tx, { companyId });
const queuedRun = await tx
.insert(heartbeatRuns)
.values({
@@ -1007,6 +1013,11 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
)).for("update");
if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company");
const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId);
const previousComments = new Set(queuedCommentIdsFromWakePayload(priorPayload));
const newComments = queuedCommentIdsFromWakePayload(input.mergedPayload).some(id => !previousComments.has(id));
const newInteraction = readNonEmptyString(input.mergedPayload.interactionId);
if (newComments || newInteraction && newInteraction !== readNonEmptyString(priorPayload.interactionId))
await notifyDeferredExecutionWork(tx, { companyId: input.companyId, requestId: prior.id });
const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId);
const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) &&
priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and(
@@ -1078,6 +1089,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
async insertNewDeferredWake(scope, input) {
const tx = requireAdmissionTx(scope, input.companyId);
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.insert(agentWakeupRequests).values({
...input.durableReceipt,
companyId: input.companyId,
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
@@ -55,6 +56,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = {
function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction {
return {
async updateCommentBody({ issueId, commentId, body, updatedAt }) {
await notifyStatusCardIssueChange(tx, companyId);
const updated = await tx
.update(issueComments)
.set({ body, updatedAt })
@@ -65,6 +67,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu
},
async touchIssueUpdatedAt({ issueId, updatedAt }) {
await notifyStatusCardIssueChange(tx, companyId);
await tx.update(issues).set({ updatedAt }).where(and(eq(issues.id, issueId), eq(issues.companyId, companyId)));
},
+5 -1
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "../services/status-card-notifications.js";
import { removePluginWorkspace } from "../services/plugin-workspaces.js";
import { canActorReadWorkspaceOperation } from "../services/heartbeat-run-privacy.js";
import { canActorReadExecutionWorkspace, executionWorkspaceReadSqlCondition } from "../services/authorization.js";
@@ -1245,7 +1246,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
await runtimeLeases.release({ executionWorkspaceId: existing.id, force: true });
if (existing.mode === "shared_workspace") {
await db
await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, existing.companyId);
await tx
.update(issues)
.set({
executionWorkspaceId: null,
@@ -1257,6 +1260,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
eq(issues.executionWorkspaceId, existing.id),
),
);
});
}
try {
+2
View File
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js";
import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js";
import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../services/unsafe-workspace-export-notifications.js";
@@ -16674,6 +16675,7 @@ export function issueRoutes(
}
// The click is durable fresh user intent, including when the message
// predates a failed run's stop. Keep its content and original attribution.
await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: locked.wake.id });
await tx.update(agentWakeupRequests).set({
payload: { ...readObject(locked.wake.payload), queuedCommentInterrupt: {
actorId: actor.actorId, requestedAt: new Date().toISOString(),
+16 -10
View File
@@ -16,6 +16,12 @@ import { accessService, heartbeatService, instanceSettingsService, issueService,
import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js";
import { assertCompanyAccess, getAccessibleResource, getActorInfo, hasCompanyAccess } from "./authz.js";
// Deadline provenance belongs to scheduling, not the public card contract.
function publicCard<T extends { nextEvalReason?: unknown }>(card: T) {
const { nextEvalReason: _reason, ...result } = card;
return result;
}
export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWakeupDeps } = {}) {
const router = Router();
const access = accessService(db);
@@ -144,9 +150,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
assertCompanyAccess(req, companyId);
await assertStatusCardsEnabled();
const query = listStatusCardsQuerySchema.parse(req.query);
res.json(await service.list(companyId, query.archived, {
res.json((await service.list(companyId, query.archived, {
issueReadCondition: await issueReadSqlCondition(db, req.actor),
}));
})).map(publicCard));
});
router.post("/companies/:companyId/status-cards", validate(createStatusCardSchema), async (req, res) => {
@@ -162,7 +168,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
try {
const compile = await enqueueCompile(req, card.id);
await logMutation(req, companyId, "status_card.created", card.id, { state: card.state });
res.status(201).json(compile.card);
res.status(201).json(publicCard(compile.card));
} catch (error) {
await service.remove(card.id);
throw error;
@@ -173,9 +179,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
await assertStatusCardsEnabled();
const card = await getAccessibleResource(req, res, service.getById(req.params.id as string), "Status card not found");
if (!card) return;
res.json(await service.hydrate(card, {
res.json(publicCard(await service.hydrate(card, {
issueReadCondition: await issueReadSqlCondition(db, req.actor),
}));
})));
});
router.patch("/status-cards/:id", validate(patchStatusCardSchema), async (req, res) => {
@@ -197,7 +203,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
fields: Object.keys(req.body),
archived: Boolean(updated.archivedAt),
});
res.json(compile?.card ?? restore?.card ?? updated);
res.json(publicCard(compile?.card ?? restore?.card ?? updated));
});
router.delete("/status-cards/:id", async (req, res) => {
@@ -234,7 +240,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
generatingIssueId: result.generatingIssue.id,
alreadyGenerating: result.alreadyGenerating,
});
res.status(result.alreadyGenerating ? 200 : 202).json(result);
res.status(result.alreadyGenerating ? 200 : 202).json({ ...result, card: publicCard(result.card) });
});
router.post("/status-cards/:id/refresh", validate(refreshStatusCardSchema), async (req, res) => {
@@ -249,7 +255,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
alreadyGenerating: result.alreadyGenerating,
enqueued: result.enqueued,
});
res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json(result);
res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json({ ...result, card: publicCard(result.card) });
});
router.get("/status-cards/:id/dry-run", async (req, res) => {
@@ -289,7 +295,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
generationIssueId: req.body.generationIssueId,
changeSummary: req.body.changeSummary,
});
res.json(updated);
res.json(publicCard(updated));
});
router.put("/status-cards/:id/summary", validate(writeStatusCardSummarySchema), async (req, res) => {
@@ -309,7 +315,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
documentId: result.document.id,
changeSummary: req.body.changeSummary,
});
res.json(result);
res.json({ ...result, card: publicCard(result.card) });
});
return router;
+4
View File
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -743,8 +745,10 @@ export function accessService(db: Db) {
sql`${issues.status} not in ('done', 'cancelled')`,
);
await notifyNativeStatusReassessment(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
const resetInProgress = await tx
.update(issues)
.set({
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { and, eq, sql } from "drizzle-orm";
import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db";
import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared";
@@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {
if (!reason.trim()) throw conflict("A repair reason is required");
const before = await inspect(tx, companyId);
if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing");
if (before.findings.some(f => f.repairable && f.kind === "agent_projection"))
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
for (const finding of before.findings.filter(f => f.repairable)) {
if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId));
if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId)));
@@ -1,3 +1,6 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
@@ -16,6 +19,7 @@ import { clearPrimaryAgent } from "./primary-agent.js";
import { budgetServiceInTransaction } from "./budgets.js";
export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise<AgentConfigurationPatch> {
await notifyAttentionRetentionWork(db, { companyId: existing.companyId });
const id = existing.id;
const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db);
if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) {
@@ -83,6 +87,8 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
existing.adapterConfig,
);
}
await notifyRunDispatch(db, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
}
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
@@ -269,6 +271,7 @@ export async function prepareConversationTurn(
context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId;
}
}
await notifyStatusCardIssueChange(tx, issue.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await tx
@@ -389,6 +392,7 @@ export async function settleConversationTurn(
issue.conversationState === conversationState
)
return true;
await notifyStatusCardIssueChange(tx, issue.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id });
@@ -544,6 +548,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo
eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId),
)).for("update");
if (!isConversation(issue) || comment.conversationSessionGeneration != null) return;
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const released = await tx.update(issueTreeHolds).set({
status: "released", releasedAt: new Date(), updatedAt: new Date(),
releasedByActorType: "user", releasedByUserId: comment.authorUserId,
+4
View File
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -34,10 +36,12 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str
if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", {
code: "agent_prompt_injection_accounting_pending",
});
await notifyAttentionRetentionWork(tx, { companyId });
await notifyExternalConversationSettlement(tx, { companyId });
await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id));
await tx
.update(issues)
+2
View File
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared";
import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils";
@@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js";
import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData> {
await notifyAttentionRetentionWork(db, { companyId });
const { runner, ...data } = input;
const selection = options?.runnerResolved
? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} }
+9 -1
View File
@@ -1,3 +1,6 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
import { isAgentStatusInvokable } from "@paperclipai/shared";
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
@@ -38,7 +41,12 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
},
notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle),
async beforeStatusChange(tx, companyId, agentId, before, after) {
if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyAttentionRetentionWork(tx, { companyId });
if (before !== after) {
await notifyRunDispatch(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
}
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
await notifyNativeResponsePresentation(tx, { companyId, agentId });
await notifyNativeReviewResponse(tx, { companyId, agentId });
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { and, eq, inArray } from "drizzle-orm";
import { agents, projects, type Db } from "@paperclipai/db";
import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared";
@@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c
const added = requested.filter(id => !previous.includes(id));
const next = [...previous, ...added];
if (added.length) {
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: {
...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next },
} }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
+5 -1
View File
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { type BudgetServiceHooks } from "./budgets.js";
import { and, desc, eq, inArray } from "drizzle-orm";
@@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
});
}
const updated = await db
const updated = await db.transaction(async tx => {
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
return tx
.update(agents)
.set({
permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }),
@@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
.where(eq(agents.id, id))
.returning()
.then((rows) => rows[0] ?? null);
});
return updated ? getById(updated.id) : null;
},
+320
View File
@@ -0,0 +1,320 @@
import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm";
import { agents, decisionRetention, decisions, heartbeatRuns,
issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db";
import { evaluateAgentInvokability } from "./agent-invokability.js";
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
import { attentionIssueCondition } from "./issue-visibility.js";
import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js";
import { issueService } from "./issues.js";
import { parseIssueExecutionState } from "./issue-execution-policy.js";
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
import type { AttentionRetentionScope } from "./retention-work-notifications.js";
const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review",
"failed_run", "budget_alert", "agent_error_alert"] as const;
type Kind = typeof kinds[number];
type Scope = AttentionRetentionScope;
type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null };
type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" };
type Cursor = { sourceKind: string; sourceId: string };
type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number;
blockedActivityAt: string | null; blockedDeadlineAt: number | null };
type PageEvidence = {
decisions: Map<string, { originIssueId: string }>;
recoveries: Map<string, typeof issueRecoveryActions.$inferSelect>;
runs: Map<string, typeof heartbeatRuns.$inferSelect>;
agents: Set<string>;
visibleIssues: Map<string, PageIssue>;
newerRuns: Set<string>;
deferredRoots: Set<string>;
blockers: Awaited<ReturnType<ReturnType<typeof issueService>["listBlockerAttention"]>>;
};
const sourceTables: Record<Kind, SQL> = {
decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`,
recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`,
failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`,
};
const sources: Record<Kind, SQL> = {
decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id
from decisions d where d.status = 'open'`,
issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text
from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`,
recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text
from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`,
blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text
from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`,
review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text
from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`,
failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId')
from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists (
select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id
and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`,
budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b
where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0
and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`,
agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`,
};
export function attentionRetentionService(db: Db) {
const cursors = new Map<string, Cursor>();
async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise<Candidate[]> {
const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []);
const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind)
&& (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind)));
if (!selected.length) return [];
const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)})
select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId",
s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt"
from source_candidates s inner join companies c on c.id = s.company_id
left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id
where c.status = 'active'
${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``}
and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid)
and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null})
and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null))
${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``}
order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[];
return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) }));
}
async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) {
const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId),
attentionIssueCondition(conversations)));
return row;
}
async function interactionEligibility(database: Db, candidates: Candidate[]) {
const result = new Map<string, boolean>();
const byCompany = new Map<string, string[]>();
for (const candidate of candidates) {
if (candidate.sourceKind !== "issue_thread_interaction") continue;
const ids = byCompany.get(candidate.companyId) ?? [];
ids.push(candidate.sourceId);
byCompany.set(candidate.companyId, ids);
}
for (const [companyId, ids] of byCompany) {
const rows = await database.select().from(issueThreadInteractions).where(and(
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids),
eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId)));
if (!rows.length) continue;
const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId),
inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true)));
const visibleIds = new Set(visible.map(row => row.id));
const roster = await database.select().from(agents).where(eq(agents.companyId, companyId));
const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId
|| !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable;
const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId);
const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and(
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds),
eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"),
isNull(issueThreadInteractions.addresseeUserId))) : [];
const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id));
for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row)
&& (row.kind !== "request_confirmation" || newest.has(row.id)));
}
return result;
}
async function reviewEligibility(database: Db, candidates: Candidate[]) {
const result = new Map<string, boolean>();
const byCompany = new Map<string, string[]>();
for (const candidate of candidates) {
if (candidate.sourceKind !== "review") continue;
const ids = byCompany.get(candidate.companyId) ?? [];
ids.push(candidate.sourceId);
byCompany.set(candidate.companyId, ids);
}
for (const [companyId, ids] of byCompany) {
const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId),
inArray(issues.id, ids), attentionIssueCondition(false)));
const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals)
.innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and(
eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId),
inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending")));
const approved = new Set(pendingApprovals.map(row => row.issueId));
const stalled = await issueService(database).listReviewAttention(companyId, rows);
for (const row of rows) {
const state = parseIssueExecutionState(row.executionState);
result.set(row.id, requiresReviewAttention({
hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user",
hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id),
stalled: stalled.get(row.id)?.state === "stalled",
}));
}
}
return result;
}
async function pageEvidence(database: Db, candidates: Candidate[]) {
const byCompany = new Map<string, Candidate[]>();
for (const candidate of candidates) {
if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue;
const rows = byCompany.get(candidate.companyId) ?? [];
rows.push(candidate);
byCompany.set(candidate.companyId, rows);
}
const result = new Map<string, PageEvidence>();
for (const [companyId, rows] of byCompany) {
const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId);
const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention");
const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId })
.from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : [];
const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : [];
const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : [];
const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and(
eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]),
sql`${agents.status} <> 'terminated'`)) : [];
const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues),
activityAt: sql<string>`${issues.updatedAt}::text`,
deadlineAt: sql<number>`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
blockedActivityAt: sql<string | null>`${issues.blockedTransitionAt}::text`,
blockedDeadlineAt: sql<number | null>`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
}).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]),
attentionIssueCondition(false))) : [];
const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId),
...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id));
const visibleRows = await readIssues(issueIds);
const visible = new Map(visibleRows.map(row => [row.id, row]));
const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id));
const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots)
: { attention: new Map(), deferredRootIds: new Set<string>() };
const blockers: PageEvidence["blockers"] = blockerBatch.attention;
const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention)
.map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id));
for (const row of await readIssues(terminalIds)) visible.set(row.id, row);
result.set(companyId, {
decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])),
runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible,
newerRuns: await listFailedRunsWithNewerRun(database, companyId,
runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds,
});
}
return result;
}
async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map<string, boolean>, interactionAttention?: Map<string, boolean>, evidence?: PageEvidence): Promise<Entry | null> {
if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) {
return { ...candidate, rootId: candidate.sourceId, variant,
id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() };
}
let sourceId = candidate.sourceId;
let activityAt = candidate.activityAt;
let dueAt = candidate.deadlineAt;
if (candidate.sourceKind === "decision") {
const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions)
.where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0];
if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId)
: await visibleIssue(database, candidate.companyId, row.originIssueId))) return null;
} else if (candidate.sourceKind === "issue_thread_interaction") {
const attention = interactionAttention ?? await interactionEligibility(database, [candidate]);
if (!attention.get(sourceId)) return null;
} else if (candidate.sourceKind === "recovery_action") {
const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0];
if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId)
: await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null;
} else if (candidate.sourceKind === "review") {
const attention = reviewAttention ?? await reviewEligibility(database, [candidate]);
if (!attention.get(sourceId)) return null;
} else if (candidate.sourceKind === "blocker_attention") {
const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
if (!row) return null;
if (variant === "owner") {
if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null;
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at,
ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues
where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
activityAt = activity.at; dueAt = Number(activity.due);
} else {
const attention = evidence ? evidence.blockers.get(row.id) : (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id);
if (!terminalBlockerRequiresAttention(attention)) return null;
sourceId = attention.terminalBlockerIssueId ?? row.id;
if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`);
const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
if (!terminal) return null;
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at,
ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
activityAt = activity.at; dueAt = Number(activity.due);
}
const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId),
eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId)));
if (retention?.keep || retention?.archivedAt) return null;
} else if (candidate.sourceKind === "failed_run") {
const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0];
if (!run) return null;
const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId),
eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0];
const issueId = readRunIssueId(run.contextSnapshot);
if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId)
: await visibleIssue(database, candidate.companyId, issueId))) return null;
if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null;
}
return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant,
id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt };
}
async function load(scope?: Scope, signal?: AbortSignal) {
const key = JSON.stringify(scope ?? {});
const rows = await candidates(db, scope, cursors.get(key));
const entries: Entry[] = [];
const reviews = await reviewEligibility(db, rows);
const interactions = await interactionEligibility(db, rows);
const evidence = await pageEvidence(db, rows);
for (const row of rows) {
if (signal?.aborted) return { rows: null, complete: true };
for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) {
const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId));
if (entry) entries.push(entry);
}
}
if (rows.length === 100) {
const last = rows[rows.length - 1]!;
cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId });
} else cursors.delete(key);
return { rows: entries, complete: rows.length < 100 };
}
async function check(entry: Entry) {
return db.transaction(async transaction => {
const database = transaction as unknown as Db;
await database.execute(sql`select id from ${sourceTables[entry.sourceKind]}
where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`);
const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId);
if (!candidate) return null;
const current = await eligible(database, candidate, entry.variant, true);
if (!current || current.dueAt > Date.now()) return current;
const retention = decisionRetentionService(database);
const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt };
const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!;
await retention.autoArchive({ companyId: current.companyId, items: [{ ...item,
keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] });
return null;
});
}
return { load, check };
}
export function registerAttentionRetentionWork(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
service: ReturnType<typeof attentionRetentionService>, canRun: () => boolean) {
return registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.attentionRetention,
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope,
matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId)
&& (!scope?.sourceKind || scope.sourceKind === entry.sourceKind)
&& (!scope?.sourceId || scope.sourceId === entry.sourceId)
&& (!scope?.issueId || scope.issueId === entry.issueId)
&& (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)),
load: service.load, check: service.check,
}, canRun);
}
@@ -0,0 +1,112 @@
import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm";
import { heartbeatRuns, type Db } from "@paperclipai/db";
export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
export const ATTENTION_BUDGET_WARNING_PERCENT = 85;
export const ATTENTION_AGENT_ERROR_STATUS = "error";
export function collapsePendingConfirmationsToNewest<T extends {
id: string;
issueId: string;
kind: string;
createdAt: Date;
payload: unknown;
}>(rows: T[]) {
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
const newestByGroup = new Map<string, T>();
for (const row of rows) {
if (row.kind !== "request_confirmation" || independent(row)) continue;
const groupKey = `${row.issueId}:${row.kind}`;
const newest = newestByGroup.get(groupKey);
if (
!newest
|| row.createdAt.getTime() > newest.createdAt.getTime()
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
) {
newestByGroup.set(groupKey, row);
}
}
return rows.filter((row) => (
row.kind !== "request_confirmation"
|| independent(row)
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
));
}
export function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
}
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
const withNewerRun = new Set<string>();
const keyed = failedRuns.filter(run => run.issueId !== null);
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
}
const unkeyed = failedRuns.filter(run => run.issueId === null);
if (unkeyed.length === 0) return withNewerRun;
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
unkeyed[0]!.createdAt);
const latestByAgent = new Map<string, Date>();
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
const rows = await db.select({
agentId: heartbeatRuns.agentId,
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
}).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId),
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
gt(heartbeatRuns.createdAt, oldestCreatedAt),
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
)).groupBy(heartbeatRuns.agentId);
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
}
for (const run of unkeyed) {
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
}
return withNewerRun;
}
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
if (failedRuns.length === 0) return [];
const failed = sql.join(
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
sql`, `,
);
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
const newerRunExists = (match: SQL) => sql`exists (
select 1 from ${heartbeatRuns}
where ${heartbeatRuns.companyId} = ${companyId}
and ${heartbeatRuns.agentId} = failed.agent_id
and ${heartbeatRuns.createdAt} >= failed.newer_from
and ${match}
)`;
const rows = await db.execute(sql`
select failed.id
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
`) as unknown as Array<{ id: string }>;
return rows.map((row) => row.id);
}
export function requiresReviewAttention(input: {
hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean;
}) {
return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled;
}
export function terminalBlockerRequiresAttention<T extends { state?: string; blockingTreeLive?: boolean }>(
attention: T | null | undefined,
): attention is T & { state: "stalled" | "needs_attention" } {
return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? ""));
}
+9 -106
View File
@@ -1,5 +1,6 @@
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js";
import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
agents,
@@ -58,7 +59,7 @@ import {
BLOCKER_ATTENTION_MAX_NODES,
issueService,
} from "./issues.js";
import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js";
import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js";
import { parseIssueExecutionState } from "./issue-execution-policy.js";
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js";
@@ -107,7 +108,6 @@ const SOURCE_RANK: Record<AttentionSourceKind, number> = {
const PENDING_INTERACTION_STATUSES = ["pending"] as const;
const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const;
const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
const DETAIL_EXCERPT_LENGTH = 160;
const DETAIL_IMAGE_LIMIT = 3;
const OPEN_DECISION_DEFAULT_LIMIT = 500;
@@ -766,37 +766,6 @@ export function interactionResolverAudience(
};
}
function collapsePendingConfirmationsToNewest<T extends {
id: string;
issueId: string;
kind: string;
createdAt: Date;
payload: unknown;
}>(rows: T[]) {
// Quality gates bind independent outputs; a newer visual approval must not
// hide a still-pending copy approval (or affect its badge count).
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
const newestByGroup = new Map<string, T>();
for (const row of rows) {
if (row.kind !== "request_confirmation" || independent(row)) continue;
const groupKey = `${row.issueId}:${row.kind}`;
const newest = newestByGroup.get(groupKey);
if (
!newest
|| row.createdAt.getTime() > newest.createdAt.getTime()
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
) {
newestByGroup.set(groupKey, row);
}
}
return rows.filter((row) => (
row.kind !== "request_confirmation"
|| independent(row)
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
));
}
function budgetObservedPercent(amountObserved: number, amountLimit: number) {
return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0;
}
@@ -866,9 +835,7 @@ async function issueSummaryMap(
.where(and(
eq(issues.companyId, companyId),
inArray(issues.id, ids),
includeConversations
? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))
: executionIssueCondition(),
attentionIssueCondition(includeConversations),
readCondition,
));
return new Map(rows.map((row) => [row.id, {
@@ -1108,70 +1075,6 @@ function resolveBlockingIssue(
return null;
}
function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
}
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
const withNewerRun = new Set<string>();
const keyed = failedRuns.filter(run => run.issueId !== null);
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
}
const unkeyed = failedRuns.filter(run => run.issueId === null);
if (unkeyed.length === 0) return withNewerRun;
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
unkeyed[0]!.createdAt);
const latestByAgent = new Map<string, Date>();
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
const rows = await db.select({
agentId: heartbeatRuns.agentId,
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
}).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId),
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
gt(heartbeatRuns.createdAt, oldestCreatedAt),
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
)).groupBy(heartbeatRuns.agentId);
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
}
for (const run of unkeyed) {
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
}
return withNewerRun;
}
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
if (failedRuns.length === 0) return [];
const failed = sql.join(
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
sql`, `,
);
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
const newerRunExists = (match: SQL) => sql`exists (
select 1 from ${heartbeatRuns}
where ${heartbeatRuns.companyId} = ${companyId}
and ${heartbeatRuns.agentId} = failed.agent_id
and ${heartbeatRuns.createdAt} >= failed.newer_from
and ${match}
)`;
const rows = await db.execute(sql`
select failed.id
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
`) as unknown as Array<{ id: string }>;
return rows.map((row) => row.id);
}
export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) {
const openDecisionLimit = Math.min(
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
@@ -1636,8 +1539,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}));
}
const blockerAttention = issue.blockerAttention;
if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue;
if (blockerAttention.blockingTreeLive) continue;
if (!terminalBlockerRequiresAttention(blockerAttention)) continue;
const issueSummary = blockedIssueSummaries.get(issue.id) ?? null;
const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id;
const terminalSummary = terminalBlockerSummaries.get(terminalIssueId)
@@ -1739,7 +1641,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null;
const reviewAttention = reviewAttentionByIssueId.get(review.id);
const stalled = reviewAttention?.state === "stalled";
if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue;
if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId),
hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue;
const issue = reviewIssueMap.get(review.id);
if (!issue) continue;
const dedupKey = `review:${review.id}`;
@@ -1858,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
const budgetOverview = await budgetService(db).overview(companyId);
for (const incident of budgetOverview.activeIncidents) {
const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit);
if (incident.thresholdType !== "hard" && observedPercent < 85) continue;
if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue;
const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`;
add(createItem({
companyId,
@@ -1921,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
updatedAt: agents.updatedAt,
})
.from(agents)
.where(and(eq(agents.companyId, companyId), eq(agents.status, "error")))
.where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS)))
.orderBy(desc(agents.updatedAt), desc(agents.id));
for (const agent of erroredAgents) {
+49 -11
View File
@@ -1,6 +1,9 @@
import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyApprovalWork } from "./approval-work-notifications.js";
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { addCents, compareCents, normalizeCents } from "@paperclipai/shared";
import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -164,6 +167,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s
};
}
async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise<ScopeRecord> {
let scope: ScopeRecord;
try {
scope = await resolveScopeRecord(db, scopeType, scopeId);
} catch (error) {
if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error;
return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project",
paused: false, pauseReason: null, budgetHeld: false };
}
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
return scope;
}
// Keep a large dashboard from filling the database pool with queued reads.
async function mapBudgetReads<T, R>(rows: T[], read: (row: T) => Promise<R>): Promise<R[]> {
const result: R[] = [];
@@ -288,6 +304,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
if (policy.scopeType === "project") {
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await db
.update(projects)
.set({
@@ -301,8 +319,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
if (scope.paused) return;
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await notifyCompanyRetentionWork(db, policy.companyId);
await notifyExternalObjectRefresh(db, policy.companyId);
await notifyStatusCardIssueChange(db, policy.companyId);
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
@@ -348,6 +369,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
if (policy.scopeType === "project") {
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await db
.update(projects)
.set({
@@ -362,8 +385,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync);
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
if (scope.pauseReason !== "budget") return;
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await notifyCompanyRetentionWork(db, policy.companyId);
await notifyExternalObjectRefresh(db, policy.companyId);
await notifyStatusCardIssueChange(db, policy.companyId);
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
@@ -398,7 +424,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId);
const observed = await computeObservedSpend(db, policy);
const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed;
const { windowStart: start, windowEnd: end } = observed;
@@ -469,6 +495,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
windowEnd: end,
});
await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" });
if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId);
const approval = thresholdType === "hard"
? await db
@@ -507,7 +534,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
return incident ? { incident, created: true } : null;
}
async function resolveOpenSoftIncidents(policyId: string) {
async function resolveOpenSoftIncidents(policyId: string, companyId: string) {
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
await db
.update(budgetIncidents)
.set({
@@ -526,10 +554,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
async function resolveOpenIncidentsForPolicy(
policyId: string,
companyId: string,
approvalStatus: "approved" | "rejected" | null,
decidedByUserId: string | null,
thresholdType?: BudgetThresholdType,
) {
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"),
thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined);
const openRows = await db
@@ -563,7 +593,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
return mapBudgetReads(rows, async (row) => {
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId);
return {
id: row.id,
companyId: row.companyId,
@@ -618,9 +648,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy;
// A closed UTC window, inactive policy, or budget raise releases only
// budget-owned pauses. Manual pauses remain operator decisions.
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard");
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard");
if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) {
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft");
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft");
}
}
// Agent status has its own admission gate and error code. Preserve a
@@ -677,7 +707,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
if (observedBlocks(policy, observed)) {
await resolveOpenSoftIncidents(policy.id);
await resolveOpenSoftIncidents(policy.id, policy.companyId);
const hardIncident = await createIncidentIfNeeded(policy, "hard", observed);
await pauseAndCancelScopeForBudget(policy);
if (hardIncident?.created) {
@@ -847,11 +877,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const observed = await computeObservedSpend(db, row);
if (!observedBlocks(row, observed)) {
await resumeScopeFromBudget(row);
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard");
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard");
if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) {
await createIncidentIfNeeded(row, "soft", observed);
} else {
await resolveOpenIncidentsForPolicy(row.id, null, null, "soft");
await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft");
}
} else {
const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100);
@@ -859,14 +889,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
await createIncidentIfNeeded(row, "soft", observed);
}
if (observedBlocks(row, observed)) {
await resolveOpenSoftIncidents(row.id);
await resolveOpenSoftIncidents(row.id, row.companyId);
await createIncidentIfNeeded(row, "hard", observed);
await pauseAndCancelScopeForBudget(row);
}
}
} else {
await resumeScopeFromBudget(row);
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service");
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service");
}
await recordActivity({
@@ -928,6 +958,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
if (incident.companyId !== companyId) throw notFound("Budget incident not found");
const policy = await getPolicyRow(incident.policyId);
if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) {
throw notFound("Budget scope not found");
}
const scope = input.action === "keep_paused"
? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId)
: await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
if (input.action === "raise_budget_and_resume") {
const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
const observed = await computeObservedSpend(db, policy);
@@ -940,6 +977,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
const now = new Date();
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId);
await db
.update(budgetPolicies)
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
import { createFailedRetryNotices } from "./failed-retry-notices.js";
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
// Source authorization above checked every external receipt and its owner.
// Reset only this certified unstarted batch under the task's admission lock.
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
const settle = async (status: string, result: Record<string, unknown>) => {
const [current] = await db.transaction(async (tx) => {
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
});
await notifyChatActionWork(tx, "failed_run_retry");
if (status === "processed") await notifyNativeResponsePresentation(tx, {
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
import { z } from "zod";
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
)).for("update");
if (receipt?.runId) return;
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "../../status-card-notifications.js";
import { notifyIssueRecovery } from "../../issue-recovery-notifications.js";
import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js";
import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js";
@@ -78,10 +80,12 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm
})(),
);
const effect = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId: input.endpoint.companyId });
// Replace the optimistic attachment placeholder with an explicit safe
// Paperclip-visible failure. The message link stays authoritative for
// exact provider redelivery, while no agent wakeup is enqueued for an
// empty turn that contains no usable bytes.
await notifyStatusCardIssueChange(tx, input.endpoint.companyId);
await tx
.update(issueComments)
.set({ body: visibleFailure, updatedAt: new Date() })
@@ -1,3 +1,4 @@
import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type ChatConfirmationScope = { companyId: string; issueId?: string };
@@ -10,6 +11,7 @@ export async function notifyChatConfirmationWork(tx: object, scope: ChatConfirma
/** Interaction resolution can release a callback parked while its human input
* was pending. Register on the owning transaction before its resolution write. */
export async function notifyChatConfirmationResolution(tx: object, interaction: ChatConfirmationScope & { kind: string }) {
if (interaction.kind === "request_confirmation") await notifyMergeConfirmationWork(tx, interaction);
if (["request_confirmation", "request_checkbox_confirmation", "suggest_tasks", "connection_intent"].includes(interaction.kind)) {
await notifyChatConfirmationWork(tx, interaction);
}
@@ -1,3 +1,4 @@
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyIssueReviewState } from "./issue-review-state-notifications.js";
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
import { notifyChatResponseAvailability } from "./chat-response-availability.js";
@@ -11,6 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string };
export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope);
if (scope.issueId || scope.runId) {
await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId,
runId: scope.issueId ? undefined : scope.runId });
+9
View File
@@ -1,3 +1,6 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js";
@@ -333,11 +336,14 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch);
if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
await notifyRunDispatch(tx, { companyId: id });
await notifyDeferredExecutionEligibility(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyExternalObjectRefresh(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardIssueChange(tx, id);
await notifyCompanyRetentionWork(tx, id);
await notifyChatInboundWakeNotices(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id);
@@ -490,11 +496,14 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
const wasAlreadyArchived = existing.status === "archived";
if (!wasAlreadyArchived) {
await notifyRunDispatch(tx, { companyId: id });
await notifyDeferredExecutionEligibility(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyExternalObjectRefresh(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardIssueChange(tx, id);
await notifyCompanyRetentionWork(tx, id);
await notifyChatInboundWakeNotices(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id);
+4
View File
@@ -1,4 +1,6 @@
import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js";
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
@@ -73,12 +75,14 @@ export async function deleteCompany(db: Db, id: string) {
.where(eq(companies.id, id)).for("no key update");
if (!existing) return null;
await notifyExternalObjectRefresh(tx, id);
await notifyIssueRecovery(tx, { companyId: id });
await notifyChatCompanyConfiguration(tx, id);
await chatCompanyDeletion.deleteCompanyData(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardEvaluation(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry, id);
await notifyToolReviewChange(tx, id);
await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id);
+7 -1
View File
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { createAgentLifecycle } from "./agent-lifecycle.js";
import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared";
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
@@ -6465,7 +6467,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
}
if (relationRows.length > 0) {
const relationCompanyId = targetCompany.id;
await db
await db.transaction(async tx => {
await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId });
await notifyIssueRecovery(tx, { companyId: relationCompanyId });
await tx
.insert(issueRelations)
.values(relationRows.map((row) => ({
companyId: relationCompanyId,
@@ -6476,6 +6481,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
createdByUserId: actorUserId ?? null,
})))
.onConflictDoNothing();
});
}
}
+22 -3
View File
@@ -207,14 +207,20 @@ function priorityRank(priority: string | null | undefined) {
return index >= 0 ? index : ISSUE_PRIORITIES.length;
}
function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
/** Duration used by saved search membership and its deadline consumers. */
export function companySearchUpdatedWithinMs(value: string | undefined): number | null {
if (!value) return null;
const match = /^(\d+)(h|d|w|m)$/.exec(value);
if (!match) return null;
const amount = Number.parseInt(match[1]!, 10);
const unit = match[2];
const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30;
return new Date(now.getTime() - hours * 60 * 60 * 1000);
return hours * 60 * 60 * 1000;
}
function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
const duration = companySearchUpdatedWithinMs(value);
return duration === null ? null : new Date(now.getTime() - duration);
}
function issueOnlyFiltersActive(query: CompanySearchQuery) {
@@ -542,7 +548,7 @@ export function companySearchService(db: Db) {
search: async (
companyId: string,
query: CompanySearchQuery,
options?: { issueReadCondition?: SQL<boolean>; projectReadCondition?: SQL<boolean> },
options?: { issueReadCondition?: SQL<boolean>; projectReadCondition?: SQL<boolean>; onNextMembershipChange?: (at: Date | null) => void },
): Promise<CompanySearchResponse> => {
const taskSearch = parseTaskSearch(query.q);
const normalizedQuery = taskSearch.normalizedQuery;
@@ -553,6 +559,7 @@ export function companySearchService(db: Db) {
const limit = query.limit;
const offset = query.offset;
if (!hasSearchText && !issueOnlyFiltersActive(query)) {
options?.onNextMembershipChange?.(null);
return {
query: query.q,
normalizedQuery,
@@ -721,6 +728,13 @@ export function companySearchService(db: Db) {
}
}
const membershipWindow = companySearchUpdatedWithinMs(query.updatedWithin);
if (options?.onNextMembershipChange && membershipWindow !== null && scopeIncludesIssues(scope)) {
// Includes rows outside the visible page: their expiry can move an
// offset/priority-ranked result into the watched set without a write.
branches.push(sql`SELECT 'membership-expiry' AS kind, min(m.updated_at)::text AS value,
0::int AS count ${countTail} FROM matched m ${branchWhere([scopeCond, ...facetsAll])}`);
}
const resultRows = await db.execute(sql`
${taskSearchCtes(companyId, taskSearch, scope !== "issues", and(...issueFilters), options?.issueReadCondition)}
${sql.join(branches, sql` UNION ALL `)}
@@ -732,6 +746,7 @@ export function companySearchService(db: Db) {
totals: { current: 0, unfiltered: 0, omit: {} },
};
const issueRowsRaw: IssueSearchRow[] = [];
let nextMembershipChange: Date | null = null;
for (const row of resultRows) {
if (row.kind === "result") {
issueRowsRaw.push({
@@ -756,6 +771,9 @@ export function companySearchService(db: Db) {
});
continue;
}
if (row.kind === "membership-expiry" && row.value && membershipWindow !== null) {
nextMembershipChange = new Date(new Date(row.value).getTime() + membershipWindow + 1);
}
const count = Number(row.count ?? 0);
if (row.kind === "type:issue") aggregates.typeCounts.issue = count;
else if (row.kind === "type:comment") aggregates.typeCounts.comment = count;
@@ -776,6 +794,7 @@ export function companySearchService(db: Db) {
aggregates.totals.omit[row.kind.slice("total:omit:".length) as CompanySearchIssueFilterKey] = count;
}
}
options?.onNextMembershipChange?.(nextMembershipChange);
return { rows: await enrichIssueSnippets(issueRowsRaw), aggregates };
}
+2
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { skillExecutionContent } from "./skill-execution-content.js";
import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js";
import { managedSkillContent, type SkillContentPublication } from "./managed-skill-content.js";
@@ -7340,6 +7341,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
for (const row of rows) {
await db.transaction(async (tx) => {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
await tx
.update(issues)
.set({ hiddenAt: now, updatedAt: now })
+8 -5
View File
@@ -26,7 +26,7 @@ export function registerDeadlineWork(
pausePassiveMaintenance: true,
onSettled: hints => { requested++; task.onSettled?.(hints); },
run: async signal => {
if (!task.canRun()) { requested++; next = null; return; }
if (!task.canRun()) { requested++; return; }
if (Date.now() < earliest) { next = earliest; return; }
if (Date.now() < coalesceUntil && (next === null || next > Date.now())) {
next = next === null ? coalesceUntil : Math.min(next, coalesceUntil);
@@ -39,8 +39,6 @@ export function registerDeadlineWork(
if (signal.aborted || !task.canRun()) { next = null; return; }
const deadline = await task.next();
if (!task.canRun()) { next = null; return; }
earliest = deadline === null || deadline > Date.now()
? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0)
@@ -49,8 +47,13 @@ export function registerDeadlineWork(
coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0);
reconciled = generation;
},
hasPending: async () => requested !== reconciled,
nextRunAt: () => next,
// A settled producer must release its admission hold during drain. Keep
// the unreconciled generation and an in-memory resume wake, without asking
// the domain to probe its database while dispatch is closed.
hasPending: async () => task.canRun() && requested !== reconciled,
nextRunAt: () => !task.canRun() && requested !== reconciled
? Date.now() + task.retryMs
: next,
});
return { ...worker, wake: () => { requested++; worker.wake(); } };
}
+1 -1
View File
@@ -23,7 +23,7 @@ export function decisionExpiryService(db: Db, deliverContinuation: (id: string)
agentId: row.originAgentId, runId: row.originRunId, action: "decision.expired", entityType: "decision",
entityId: row.id, details: { expiredReason: reason } }, publications);
return row;
});
}, { companyId: decision.companyId, sourceId: decision.id });
if (!updated) return 0;
for (const publication of publications) publishActivity(publication);
await deliverContinuation(updated.id);
+29 -14
View File
@@ -1,3 +1,4 @@
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
import { notifyRetentionWork } from "./retention-work-notifications.js";
import { decisionArchiveDeliveryService, type ArchiveNotificationBatch } from "./decision-archive-delivery.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -15,7 +16,6 @@ import {
issueRecoveryActions,
issueThreadInteractions,
issues,
joinRequests,
} from "@paperclipai/db";
import type {
AttentionArchiveManifestEntry,
@@ -145,7 +145,7 @@ export function decisionRetentionService(
const existing = new Set(rows.map(row => sourceKey(row.sourceKind, row.sourceId)));
const missing = unique.filter(item => !existing.has(itemSourceKey(item)));
if (missing.length) {
for (const kind of new Set(missing.map(item => item.sourceKind))) await notifyRetentionWork(tx, kind);
for (const item of missing) await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id });
await tx.insert(decisionRetention).values(missing.map(item => ({
companyId, sourceKind: item.sourceKind, sourceId: item.subject.id, sourceActivityAt: new Date(item.activityAt),
}))).onConflictDoNothing({ target: [decisionRetention.companyId, decisionRetention.sourceKind, decisionRetention.sourceId] });
@@ -155,7 +155,7 @@ export function decisionRetentionService(
for (const item of unique) {
const key = itemSourceKey(item), row = byKey.get(key), activityAt = new Date(item.activityAt);
if (!row || row.sourceActivityAt.getTime() === activityAt.getTime()) continue;
await notifyRetentionWork(tx, item.sourceKind);
await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id });
const [updated] = await tx.update(decisionRetention).set({
sourceActivityAt: activityAt, version: sql`${decisionRetention.version} + 1`, updatedAt: new Date(),
}).where(and(eq(decisionRetention.id, row.id), ne(decisionRetention.sourceActivityAt, activityAt))).returning();
@@ -169,9 +169,9 @@ export function decisionRetentionService(
return sourceKey(item.sourceKind, item.subject.id);
}
async function writeRetention<T>(kind: AttentionSourceKind, write: (tx: Db) => Promise<T>) {
async function writeRetention<T>(companyId: string, kind: AttentionSourceKind, sourceId: string, write: (tx: Db) => Promise<T>) {
return db.transaction(async tx => {
await notifyRetentionWork(tx, kind);
await notifyRetentionWork(tx, kind, { companyId, sourceId });
return write(tx as unknown as Db);
});
}
@@ -185,13 +185,28 @@ export function decisionRetentionService(
}
async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string }) {
const table = input.sourceKind === "approval" ? approvals : input.sourceKind === "join_request" ? joinRequests : null;
if (!table) return;
const [source] = await database.select({ updatedAt: table.updatedAt }).from(table)
.where(and(eq(table.companyId, input.companyId), eq(table.id, input.sourceId))).for("update");
const tables: Record<AttentionSourceKind, string> = {
approval: "approvals", join_request: "join_requests", decision: "decisions",
issue_thread_interaction: "issue_thread_interactions", recovery_action: "issue_recovery_actions",
failed_run: "heartbeat_runs", productivity_review: "issues", blocker_attention: "issues", review: "issues",
budget_alert: "budget_incidents", agent_error_alert: "agents",
};
if (input.sourceKind === "blocker_attention") {
const [source] = await database.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceId))).for("update");
if (!source) throw notFound("Attention source not found");
const activityAt = source.unblockDescriptor?.owner === "board" && isProspectiveBlockedTransition(source)
? source.blockedTransitionAt : source.updatedAt;
await decisionRetentionService(database).syncItems(input.companyId, [{
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: activityAt.toISOString(),
}]);
return;
}
const activity = input.sourceKind === "failed_run" ? sql`coalesce(finished_at, updated_at, created_at)` : sql`updated_at`;
const [source] = await database.execute(sql`select ${activity} as "activityAt" from ${sql.identifier(tables[input.sourceKind])}
where company_id = ${input.companyId}::uuid and id = ${input.sourceId}::uuid for update`) as unknown as Array<{ activityAt: Date | string }>;
if (!source) throw notFound("Attention source not found");
await decisionRetentionService(database).syncItems(input.companyId, [{
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(),
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: new Date(source.activityAt).toISOString(),
}]);
}
@@ -206,7 +221,7 @@ export function decisionRetentionService(
if (!(await canReadDecisionSource(db, input.authActor, input.companyId, input.sourceKind, input.sourceId))) {
throw notFound("Attention source not found");
}
const updated = await writeRetention(input.sourceKind, async tx => {
const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => {
await materializeSourceState(tx, input);
return tx.update(decisionRetention).set({
keep: input.keep,
@@ -247,7 +262,7 @@ export function decisionRetentionService(
throw notFound("Attention source not found");
}
const now = new Date();
const updated = await writeRetention(input.sourceKind, async tx => {
const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => {
await materializeSourceState(tx, input);
return tx.update(decisionRetention).set(input.archived ? {
archivedAt: now,
@@ -324,7 +339,7 @@ export function decisionRetentionService(
}
const now = new Date();
for (const entry of input.manifest) {
await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind);
await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind, { companyId: input.companyId, sourceId: entry.sourceId });
const archived = await txDb.update(decisionRetention).set({
archivedAt: now,
archivedReason: "accepted_proposal",
@@ -368,7 +383,7 @@ export function decisionRetentionService(
await db.transaction(async (tx) => {
const txDb = tx as unknown as Db;
for (const item of candidates) {
await notifyRetentionWork(txDb, item.sourceKind);
await notifyRetentionWork(txDb, item.sourceKind, { companyId: input.companyId, sourceId: item.subject.id });
const updated = await txDb.update(decisionRetention).set({
archivedAt: now,
archivedReason: "idle_ttl",
@@ -1,18 +1,23 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import type { Db } from "@paperclipai/db";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export function notifyDecisionCreated(transaction: object) {
type DecisionWorkScope = { companyId: string; sourceId?: string };
export async function notifyDecisionCreated(transaction: object, scope: DecisionWorkScope) {
await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" });
return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry);
}
export async function notifyDecisionChange(transaction: object) {
export async function notifyDecisionChange(transaction: object, scope: DecisionWorkScope) {
await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" });
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry);
}
export function writeDecisionChange<T>(db: Db, write: (transaction: Db) => Promise<T>): Promise<T> {
export function writeDecisionChange<T>(db: Db, write: (transaction: Db) => Promise<T>, scope: DecisionWorkScope): Promise<T> {
return db.transaction(async transaction => {
await notifyDecisionChange(transaction);
await notifyDecisionChange(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery);
return write(transaction as unknown as Db);
});
+13 -12
View File
@@ -239,7 +239,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
...(input.additionalTargetSnapshots ?? {}),
};
const id = randomUUID();
await notifyDecisionCreated(dbOrTx);
await notifyDecisionCreated(dbOrTx, { companyId: input.companyId, sourceId: id });
await notifyDecisionTargetWork(dbOrTx, `decision:${id}`);
const [created] = await dbOrTx.insert(decisions).values({ id, companyId: input.companyId, bundleId: input.bundleId ?? null,
originAgentId: input.agentId, originIssueId: provenance.issueId, originRunId: input.runId, ruleKey: input.ruleKey ?? null,
@@ -581,7 +581,8 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
return attempt;
}
async function settleInvalidEffectOutcome(id: string) {
async function settleInvalidEffectOutcome(decision: typeof decisions.$inferSelect) {
const id = decision.id;
const publications: ActivityPublication[] = [];
await writeDecisionChange(db, async tx => {
const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update");
@@ -601,14 +602,14 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
action: "decision.recovery_failed", entityType: "decision", entityId: id,
details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId },
}, publications);
});
}, { companyId: decision.companyId, sourceId: id });
for (const publication of publications) publishActivity(publication);
return outcome(id);
}
async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) {
const option = decision.options.find((item) => item.id === decision.chosenOptionId);
if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id);
if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision);
const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null);
const metadata = decision.metadata as Record<string, unknown>;
if (metadata.kind === "attention_archive_proposal") {
@@ -618,7 +619,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
targetSnapshots: decision.targetSnapshots as Record<string, Snapshot>,
}), decision.signedSpec)) {
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "invalid_signature", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
if (option.id === "archive") {
@@ -629,7 +630,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
const manifestHash = manifest ? hashAttentionArchiveManifest(manifest) : null;
if (!manifest || manifestHash !== metadata.manifestHash) {
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "manifest_mismatch", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
const originActor: AuthorizationActor = {
@@ -651,7 +652,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
} catch (error) {
const message = error instanceof Error ? error.message : "archive_proposal_failed";
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: message, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
}
@@ -659,7 +660,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
executionStatus: "succeeded",
updatedAt: new Date(),
metadata: { ...metadata, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) },
}).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
}).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
for (let index = 0; index < option.effects.length; index += 1) await executeEffect(decision, option.effects[index]!, index, userActor, decision.decidedByUserId, run?.responsibleUserId ?? null);
@@ -667,7 +668,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
const successful = rows.filter((row) => row.status === "executed").length;
const status = rows.every((row) => row.status === "executed") ? "succeeded" : successful ? "partial" : "failed";
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: status, updatedAt: new Date(), metadata: { ...decision.metadata,
...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
@@ -706,7 +707,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
decideIdempotencyKey: input.idempotencyKey ?? null,
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}),
...(input.dismissed ? { dismissed: true, dismissReason: input.dismissReason ?? null } : {}) } })
.where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning());
.where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: current.id });
if (!claimed) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed.originRunId)).then((rows) => rows[0] ?? null);
await logActivity(db, { companyId: claimed.companyId, actorType: "system", actorId: "decision-executor", agentId: claimed.originAgentId, runId: claimed.originRunId,
@@ -724,7 +725,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
return (await get(id))!;
}
const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "cancelled", updatedAt: new Date(), metadata: { ...current.metadata,
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning());
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id });
if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
await logActivity(db, { companyId: updated.companyId, actorType: actor.actorType, actorId: actor.actorId, runId: actor.runId, action: "decision.cancelled", entityType: "decision", entityId: id });
await continuations.deliver(updated.id);
@@ -738,7 +739,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
if (empty) return decide({ id, optionId: empty.id, decidedByUserId: userId, userActor, dismissed: true, dismissReason: reason });
const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "succeeded", chosenOptionId: "dismissed", decidedByUserId: userId,
decidedAt: new Date(), updatedAt: new Date(), metadata: { ...current.metadata, dismissed: true, dismissReason: reason ?? null,
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning());
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id });
if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
await logActivity(db, { companyId: updated.companyId, actorType: "system", actorId: "decision-executor", agentId: updated.originAgentId,
runId: updated.originRunId, responsibleUserIdOverride: userId, action: "decision.dismissed", entityType: "decision", entityId: updated.id,
@@ -0,0 +1,34 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type DeferredExecutionScope = { companyId: string; requestId?: string };
export type DeferredReceiptRecoveryOptions = {
companyId?: string;
requestId?: string;
onError?: (error: unknown) => void;
};
function hint(scope: DeferredExecutionScope) {
return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
}
export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
}
export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
}
export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
}
export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
if (scope) {
await notifyDeferredExecutionWork(tx, scope);
await notifyNativeStatusWakeWork(tx, scope);
} else {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
}
}
@@ -0,0 +1,24 @@
import { and, eq, sql } from "drizzle-orm";
import { agentWakeupRequests, type Db } from "@paperclipai/db";
import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
return db.transaction(async tx => {
const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
.where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
if (!current) return;
const previous = current.payload?.executionWait;
const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
? (previous as Record<string, unknown>).reason : undefined;
if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
await tx.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
|| ${JSON.stringify(condition)}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
});
}
@@ -2,6 +2,18 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
/** Topics for existing durable queues; register intent before their writes. */
export const DELIVERY_QUEUES = {
deferredExecutionReceipts: "deferred-execution-receipts",
taskRetryReceiptSettlement: "task-retry-receipt-settlement",
nativeStatusWakeIntents: "native-status-wake-intents",
orphanBlockers: "orphan-blockers",
activeRecoveryActions: "active-recovery-actions",
strandedAssignments: "stranded-assignments",
resolvedDependencies: "resolved-dependencies",
taskWatchdogs: "task-watchdogs",
staleIssueLocks: "stale-issue-locks",
scheduledRetryDispatch: "scheduled-retry-dispatch",
queuedRunDispatch: "queued-run-dispatch",
statusCardEvaluation: "status-card-evaluation",
runEvidenceMigration: "run-evidence-migration",
providerTraceRetention: "provider-trace-retention",
nativeRunnerEvidenceRetention: "native-runner-evidence-retention",
@@ -36,6 +48,7 @@ export const DELIVERY_QUEUES = {
executionDisposition: "execution-disposition",
environmentCleanup: "environment-cleanup",
approvalQueueSeeding: "approval-queue-seeding",
attentionRetention: "attention-retention",
approvalRetention: "approval-retention",
joinRequestRetention: "join-request-retention",
decisionTargets: "decision-targets",
@@ -98,6 +111,7 @@ export const DELIVERY_QUEUES = {
browser: "browser-use",
email: "email-channels",
museMaintenance: "muse-maintenance",
mergedPullRequestConfirmations: "merged-pull-request-confirmations",
mcpEvents: "mcp-events",
dotEvents: "dot-events",
grokDispatch: "grok-dispatch",
@@ -115,6 +129,7 @@ export const DELIVERY_QUEUES = {
pluginLifecycle: "plugin-resource-lifecycle",
toolAction: "tool-action-receipts",
terminalWorkspaces: "terminal-workspaces",
routineExecutionStatus: "routine-execution-status",
routineSchedules: "routine-schedules",
issueMonitors: "issue-monitors",
agentTimers: "agent-timers",
+16
View File
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { randomUUID } from "node:crypto";
import { and, asc, desc, eq, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -238,6 +240,7 @@ export function documentService(db: Db) {
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
try {
return await db.transaction(async (tx) => {
await notifyStatusCardIssueChange(tx, issue.companyId);
const now = new Date();
const existing = await tx
.select({
@@ -275,6 +278,7 @@ export function documentService(db: Db) {
.where(eq(issueDocuments.issueId, issue.id));
const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key));
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const [document] = await tx
.insert(documents)
.values({
@@ -376,6 +380,7 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const nextRevisionNumber = existing.latestRevisionNumber + 1;
const [revision] = await tx
.insert(documentRevisions)
@@ -438,6 +443,7 @@ export function documentService(db: Db) {
throw conflict("Document does not exist yet", { key });
}
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const [document] = await tx
.insert(documents)
.values({
@@ -592,6 +598,10 @@ export function documentService(db: Db) {
});
}
await db.transaction(async (tx) => {
for (const companyId of new Set(rows.map(row => row.companyId))) {
await notifyStatusCardIssueChange(tx, companyId);
await notifyIssueRecovery(tx, { companyId });
}
await insertRowsInChunks(tx, documents, documentRows);
await insertRowsInChunks(tx, documentRevisions, revisionRows);
await insertRowsInChunks(tx, issueDocuments, linkRows);
@@ -644,6 +654,7 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
const nextRevisionNumber = existing.latestRevisionNumber + 1;
const [restoredRevision] = await tx
@@ -662,6 +673,7 @@ export function documentService(db: Db) {
})
.returning();
await notifyStatusCardIssueChange(tx, existing.companyId);
await tx
.update(documents)
.set({
@@ -722,6 +734,7 @@ export function documentService(db: Db) {
};
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
await tx
.update(documents)
@@ -769,6 +782,7 @@ export function documentService(db: Db) {
};
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
await tx
.update(documents)
@@ -817,7 +831,9 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id));
await notifyStatusCardIssueChange(tx, existing.companyId);
await tx.delete(documents).where(eq(documents.id, existing.id));
return {
+7 -1
View File
@@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s
import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js";
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js";
import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
import { createHash, randomUUID } from "node:crypto";
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
@@ -4021,7 +4022,12 @@ export function environmentRuntimeService(
const lease = await environmentsSvc.getLeaseById(leaseId);
if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined;
const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease });
// Legacy Cloud leases can lack a credential version that permits a
// provider-local observation. Cloud's manager verifies the historical
// organization and signs only exact absence; no replacement key or
// cleanup label is treated as proof. Explicit provider answers win.
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease })
?? await inspectCloudSandboxCleanupAbsence(lease);
},
async retryPendingSandboxTeardown(input: {
@@ -1,7 +1,21 @@
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import type { Db } from "@paperclipai/db";
import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js";
export async function notifyExecutionRunWork(transaction: object) {
export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string };
export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) {
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
if (scope) {
await notifyAttentionRetentionWork(transaction, scope);
await notifyStatusCardIssueChange(transaction, scope.companyId);
await notifyRunDispatch(transaction, scope);
await notifyDeferredExecutionEligibility(transaction, scope);
}
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
await notifyDeliveryWork(transaction, queue);
@@ -12,8 +26,14 @@ export function writeExecutionControl<T>(
db: Db | Parameters<Parameters<Db["transaction"]>[0]>[0],
queue: DeliveryQueue | readonly DeliveryQueue[],
write: (tx: Db) => PromiseLike<T>,
scope: ExecutionWorkScope | undefined,
): Promise<T> {
return db.transaction(async transaction => {
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
if (scope) await notifyAttentionRetentionWork(transaction, scope);
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
await notifyRunDispatch(transaction, scope);
if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
return write(transaction as unknown as Db);
@@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl(
await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id });
await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id });
await notifyUnsafeWorkspaceExportRunChange(tx, run);
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: run.companyId });
const [updatedRun] = await tx
.update(heartbeatRuns)
.set({
@@ -1,3 +1,4 @@
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { notifyChatResponseAvailability } from "./chat-response-availability.js";
@@ -546,7 +547,7 @@ export async function settleUnrecoverableExecutions(
companyId: run.companyId, issueId: task.id, runId: run.id,
}).limit(1).for("update");
if (!hold) return;
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: run.companyId });
const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions.";
await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled",
resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null,
@@ -596,6 +597,7 @@ export async function settleUnrecoverableExecutions(
if (task.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: task.companyId, issueId: task.id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, task.id, task.companyId);
const [projected] = await tx
.update(issues)
.set({
@@ -654,7 +656,7 @@ export async function settleUnrecoverableExecutions(
replay: "not_authorized",
},
});
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: run.companyId });
await tx
.update(heartbeatRuns)
.set({ executionStatusDeliveryId: randomUUID() })
+8 -1
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js";
import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js";
import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
@@ -1590,6 +1591,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
await validateSelection(input, tx);
const intent: TaskWorkspaceIntent = { version: 1, request: { key: input.requestKey, expectedBindingRevision: input.expectedBindingRevision }, selection: input.selection, source: input.source ?? "explicit" };
const nextRevision = task.workspaceBindingRevision + 1;
await notifyStatusCardIssueChange(tx, task.companyId);
await tx.update(issues).set({ workspaceBindingRevision: nextRevision,
workspacePendingSelection: { version: 1, requestKey: input.requestKey,
expectedBindingRevision: nextRevision, intent }, updatedAt: new Date() }).where(eq(issues.id, task.id));
@@ -1633,6 +1635,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
}
}
await notifyStatusCardIssueChange(tx, task.companyId);
await tx.update(issues).set({ ...patch, assigneeAdapterOverrides, workspaceSelection: pending.intent, workspacePendingSelection: null,
workspaceBindingRevision: task.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, task.id));
return true;
@@ -1651,6 +1654,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
const pendingAfterFirstBinding = !task.executionWorkspaceId && pending?.expectedBindingRevision === task.workspaceBindingRevision
? { ...pending, expectedBindingRevision: nextRevision }
: pending;
await notifyStatusCardIssueChange(tx, companyId);
const [bound] = await tx.update(issues).set({ ...patch, executionWorkspaceId: workspaceId,
workspaceBindingRevision: nextRevision, workspacePendingSelection: pendingAfterFirstBinding, updatedAt: new Date() })
.where(eq(issues.id, issueId)).returning();
@@ -2526,13 +2530,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
forceWorktreeRemoval: false,
});
if (cleanup.cleaned && workspace.mode === "shared_workspace") {
await db
await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, workspace.companyId);
await tx
.update(issues)
.set({ executionWorkspaceId: null, updatedAt: now() })
.where(and(
eq(issues.companyId, workspace.companyId),
eq(issues.executionWorkspaceId, workspace.id),
));
});
}
const cleanupReason = [ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON, ...cleanup.warnings].join(" | ");
if (!cleanup.cleaned || cleanup.warnings.length > 0) {
@@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: {
}).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId),
eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution")));
if (cancelledStartupIds.size) await db.transaction(async tx => {
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId, agentId });
for (const runId of cancelledStartupIds) {
await tx.update(nativeRunFinalizations).set({
phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null,
@@ -1,7 +1,9 @@
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js";
export function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
await notifyIssueRecovery(tx, scope);
return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement,
scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*");
}
@@ -1,5 +1,5 @@
import type { Db } from "@paperclipai/db";
import { createGitHubExternalObjectProvider } from "./github-external-object-provider.js";
import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js";
export type GitHubPullRequestReference = {
host: "github.com";
@@ -25,11 +25,13 @@ export type PullRequestMergeDetails = {
export type PullRequestMergeStateResolver = (
companyId: string,
reference: GitHubPullRequestReference,
signal?: AbortSignal,
) => Promise<PullRequestMergeState>;
export type PullRequestMergeDetailsResolver = (
companyId: string,
reference: GitHubPullRequestReference,
signal?: AbortSignal,
) => Promise<PullRequestMergeDetails>;
export const PULL_REQUEST_CACHE_MAX_ENTRIES = 1_000;
@@ -86,14 +88,15 @@ function readRecord(value: unknown): Record<string, unknown> | null {
: null;
}
export function createPullRequestMergeDetailsResolver(db: Db): PullRequestMergeDetailsResolver {
const resolver = createGitHubExternalObjectProvider(db).resolvers
export function createPullRequestMergeDetailsResolver(db: Db, options?: GitHubExternalObjectProviderOptions): PullRequestMergeDetailsResolver {
const resolver = createGitHubExternalObjectProvider(db, options).resolvers
.find((candidate) => candidate.objectType === "pull_request") ?? null;
return async (companyId, reference) => {
return async (companyId, reference, signal) => {
if (!resolver) return { state: "unknown", headRef: null, headSha: null };
const result = await resolver.resolve({
companyId,
signal,
object: {
externalId: `${reference.owner}/${reference.repo}#pull/${reference.number}`,
sanitizedCanonicalUrl: `https://github.com/${reference.owner}/${reference.repo}/pull/${reference.number}`,
@@ -123,7 +126,7 @@ export function createPullRequestMergeDetailsResolver(db: Db): PullRequestMergeD
};
}
export function createPullRequestMergeStateResolver(db: Db): PullRequestMergeStateResolver {
const resolveDetails = createPullRequestMergeDetailsResolver(db);
return async (companyId, reference) => (await resolveDetails(companyId, reference)).state;
export function createPullRequestMergeStateResolver(db: Db, options?: GitHubExternalObjectProviderOptions): PullRequestMergeStateResolver {
const resolveDetails = createPullRequestMergeDetailsResolver(db, options);
return async (companyId, reference, signal) => (await resolveDetails(companyId, reference, signal)).state;
}
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm";
import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { authorizeExternalBotOperator } from "../external-bot-authority.js";
@@ -74,6 +75,7 @@ export async function inspectGrokWorkspaceObligations(db: Db, companyId: string,
return { ...rest, externalWorkspaceInspection: inspection };
};
await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id));
if (actions.length) await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "recovery_action" });
for (const action of actions) {
await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(),
nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.",
+7 -2
View File
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
@@ -9,6 +10,8 @@ import type { Db } from "@paperclipai/db";
import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db";
import { nativeSha256 } from "./native-runtime/canonical.js";
export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted";
export interface AppendHeartbeatRunEventInput {
companyId: string;
runId: string;
@@ -74,10 +77,11 @@ export async function appendHeartbeatRunEvent(
db: Db,
input: AppendHeartbeatRunEventInput,
): Promise<AppendHeartbeatRunEventResult> {
const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX));
const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null;
const projectsProgress = isSafeNativeChatProgressEventType(input.eventType);
const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent);
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment) {
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) {
const allocated = db.$with("allocated_run_event", {}).as(sql`
update ${heartbeatRuns}
set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1,
@@ -129,7 +133,7 @@ export async function appendHeartbeatRunEvent(
eq(heartbeatRunEvents.runId, input.runId),
eq(heartbeatRunEvents.agentId, input.agentId),
eq(heartbeatRunEvents.eventType, "lifecycle"),
sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`,
sql`${heartbeatRunEvents.message} like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`}`,
sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`,
))
.limit(1)
@@ -187,6 +191,7 @@ export async function appendHeartbeatRunEvent(
if (projectsResponse && run.runtimeMode === "native") {
await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId });
}
if (projectsRetention) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "failed_run" });
const seq = await allocateHeartbeatRunEventSeq(
tx as unknown as Db,
input.runId,
+44 -56
View File
@@ -1,3 +1,7 @@
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
import { registerIssueRecoveryWork } from "./recovery/work.js";
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
@@ -999,6 +1003,7 @@ export function heartbeatService(
finalizeAiConnectionBusyDeferral,
finalizeWorkspaceBusyDeferral,
promoteDueScheduledRetries,
nextScheduledRetryAt,
retryScheduledRetryNow,
scheduleBoundedRetry,
} = createHeartbeatRetries(db, {
@@ -1333,8 +1338,8 @@ export function heartbeatService(
return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args);
}
function resumeQueuedRunsAfterDrain() {
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain();
function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
activeWakeupPromises.add(promise);
void promise.then(
() => { activeWakeupPromises.delete(promise); },
@@ -1609,15 +1614,15 @@ export function heartbeatService(
void reportRunFailure(db, updated, failureReport);
}
async function reconcileStrandedAssignedIssues(opts?: { skipExternalConversationSettlement?: boolean }) {
async function reconcileStrandedAssignedIssues(opts?: Parameters<typeof recovery.reconcileStrandedAssignedIssues>[0]) {
return recovery.reconcileStrandedAssignedIssues({
skipExternalConversationSettlement: opts?.skipExternalConversationSettlement,
...opts,
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
});
}
async function sweepStaleIssueLocks() {
return recovery.sweepStaleIssueLocks();
async function sweepStaleIssueLocks(opts?: Parameters<typeof recovery.sweepStaleIssueLocks>[0]) {
return recovery.sweepStaleIssueLocks(opts);
}
function issueIdFromRunContext(contextSnapshot: unknown) {
@@ -1647,10 +1652,7 @@ export function heartbeatService(
});
}
async function reconcileTaskWatchdogs(opts?: {
companyId?: string | null;
runId?: string | null;
}) {
async function reconcileTaskWatchdogs(opts?: Parameters<typeof taskWatchdogs.reconcileTaskWatchdogs>[0]) {
return taskWatchdogs.reconcileTaskWatchdogs({
...opts,
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
@@ -1675,10 +1677,7 @@ export function heartbeatService(
return recovery.buildRunOutputSilence(run, now);
}
async function reconcileResolvedDependencyWakes(opts?: {
runId?: string | null;
companyId?: string | null;
}) {
async function reconcileResolvedDependencyWakes(opts?: Parameters<typeof recovery.reconcileResolvedDependencyWakeBackstop>[0]) {
return recovery.reconcileResolvedDependencyWakeBackstop(opts);
}
@@ -2007,8 +2006,11 @@ export function heartbeatService(
const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner
&& typeof run.contextSnapshot?.conversationSessionGeneration === "number";
if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId))
&& !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) {
const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn
? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId })
.from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null)
: null;
if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) {
await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" });
await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() });
await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true });
@@ -2204,44 +2206,6 @@ export function heartbeatService(
context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true;
}
const wakeCommentId = deriveCommentId(context, null);
const wakeCommentContext =
issueContext && wakeCommentId
? await db
.select({
id: issueComments.id,
body: issueComments.body,
authorType: issueComments.authorType,
authorAgentId: issueComments.authorAgentId,
authorUserId: issueComments.authorUserId,
presentation: issueComments.presentation,
metadata: issueComments.metadata,
deletedAt: issueComments.deletedAt,
deletedByType: issueComments.deletedByType,
deletedByAgentId: issueComments.deletedByAgentId,
deletedByUserId: issueComments.deletedByUserId,
deletedByRunId: issueComments.deletedByRunId,
sourceTrust: issueComments.sourceTrust,
})
.from(issueComments)
.where(
and(
eq(issueComments.id, wakeCommentId),
eq(issueComments.issueId, issueContext.id),
eq(issueComments.companyId, agent.companyId),
),
)
.then((rows) => {
const row = rows[0] ?? null;
return row?.deletedAt
? {
...row,
body: "",
presentation: null,
metadata: null,
}
: row;
})
: null;
let issueAssigneeOverrides =
issueContext && issueContext.assigneeAgentId === agent.id
? parseIssueAssigneeAdapterOverrides(
@@ -2610,7 +2574,7 @@ export function heartbeatService(
continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest,
} = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, {
run, agent,
task: { issueRef, issueContext, wakeCommentContext, context, taskSession },
task: { issueRef, issueContext, context, taskSession },
policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) },
services: { issuesSvc, getPinnedSkillTestContext },
}));
@@ -3724,7 +3688,7 @@ export function heartbeatService(
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx
.update(heartbeatRuns)
.set({ executionControlDeadlineAt: null })
.where(eq(heartbeatRuns.id, run.id)));
.where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId });
}
nativeOwnershipHeld =
nativeOwnershipHeld ||
@@ -4246,6 +4210,30 @@ export function heartbeatService(
computeTaskDrain,
applyTaskDrain,
registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
registerDeferredReceiptWork(db, coordinator, {
resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
dispatchNative: dispatchPendingNativeStatusWakeups,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
registerIssueRecoveryWork(db, coordinator, {
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
orphanBlockers: recovery.reconcileUnassignedBlockingIssues,
recoveryActions: recovery.reconcileActiveRecoveryActions,
dependencies: reconcileResolvedDependencyWakes,
locks: sweepStaleIssueLocks,
watchdogs: reconcileTaskWatchdogs,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
registerRunDispatchWork: (coordinator: Parameters<typeof registerRunDispatchWork>[0], canRun: () => boolean) =>
registerRunDispatchWork(coordinator, {
promoteDueScheduledRetries,
nextScheduledRetryAt,
resumeQueuedRunsAfterDrain,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
promoteDueScheduledRetries,
retryScheduledRetryNow,
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
import { and, asc, desc, eq, lte, sql } from "drizzle-orm";
@@ -91,6 +92,7 @@ export function createHeartbeatAgentStatus(db: Db) {
)).limit(1);
const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure);
const now = new Date();
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
const [updated] = await tx.update(agents).set({
status: nextStatus,
@@ -140,6 +142,7 @@ export function createHeartbeatAgentStatus(db: Db) {
eq(heartbeatRuns.agentId, candidate.id)))
.orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1);
const nextStatus = settledStatus(lastRun?.status);
await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId });
const [agent] = await tx.update(agents).set({ status: nextStatus,
errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now,
}).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId),
@@ -0,0 +1,118 @@
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
import { agentWakeupRequests, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
type Entry = { id: string; companyId: string; dueAt: number };
type NativeIntentOptions = DeferredExecutionScope & {
onDeferredDeadline(at: number): void;
onError(error: unknown): void;
};
export function registerDeferredReceiptWork(
db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
function register(kind: "deferred" | "task_chat" | "native") {
const cursors = new Map<string, string>();
let enabled = false;
const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
: kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
load: async scope => {
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
or ${agentWakeupRequests.payload}->'executionWait' is not null`,
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
}).from(agentWakeupRequests).where(and(
scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
kind === "native" ? and(
eq(agentWakeupRequests.requestedByActorType, "system"),
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
eq(agentWakeupRequests.reason, "task_retry"),
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
)),
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
const complete = rows.length < 100;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
: Date.now(),
})), complete };
},
check: async entry => {
if (!enabled) return null;
if (kind !== "task_chat") {
const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
}).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
eq(agentWakeupRequests.id, entry.id)));
if (!current) return null;
const due = kind === "native" && current.status === "claimed" && current.claimedAt
? current.claimedAt.getTime() + 60_000
: kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
? current.updatedAt.getTime() + 30_000 : null;
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
}
const errors: unknown[] = [];
let next: number | null = null;
const scope = { companyId: entry.companyId, requestId: entry.id };
if (kind === "native") await owner.dispatchNative({ ...scope,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
onError: error => { errors.push(error); },
});
else if (kind === "task_chat") await owner.settleTaskChat(scope);
else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
return next === null ? null : { ...entry, dueAt: next };
},
}, canRun);
}
const workers = [register("deferred"), register("task_chat"), register("native")];
return { ready: Promise.all(workers.map(worker => worker.ready)),
wake() { for (const worker of workers) worker.invalidate(); },
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
}
@@ -0,0 +1,58 @@
import type { RunDispatchScope } from "../run-dispatch-notifications.js";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { registerDeadlineWork } from "../deadline-work.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
export function registerRunDispatchWork(
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
promoteDueScheduledRetries(): Promise<unknown>;
nextScheduledRetryAt(): Promise<number | null>;
resumeQueuedRunsAfterDrain(scope?: RunDispatchScope): Promise<unknown>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
let suppressed = false;
const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, {
retryMs: 30_000,
continuationMs: 1_000,
notificationCoalesceMs: 1_000,
canRun,
run: async signal => {
suppressed = await owner.schedulingSuppressed();
if (!signal.aborted && !suppressed && canRun()) await owner.promoteDueScheduledRetries();
},
next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(),
});
const scopes = new Set<string>(["*"]);
const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, {
retryMs: 30_000,
notificationCoalesceMs: 1_000,
canRun,
onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); },
run: async signal => {
const requested = scopes.has("*") ? ["*"] : [...scopes];
if (requested[0] === "*") scopes.clear();
else for (const hint of requested) scopes.delete(hint);
for (let index = 0; index < requested.length; index++) {
if (signal.aborted || !canRun()) {
for (const hint of requested.slice(index)) scopes.add(hint);
return;
}
try {
await owner.resumeQueuedRunsAfterDrain(requested[index] === "*" ? undefined : JSON.parse(requested[index]) as RunDispatchScope);
} catch (error) {
for (const hint of requested.slice(index)) scopes.add(hint);
throw error;
}
}
},
next: async () => null,
});
return {
ready: Promise.all([retries.ready, queue.ready]),
wake() { scopes.add("*"); retries.wake(); queue.wake(); },
async stop() { await Promise.all([retries.stop(), queue.stop()]); },
};
}
@@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId,
runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId };
const runScopedMentionedSkillKeys =
preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
const [mentionedSkills, githubIdentity] = await Promise.allSettled([
preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
db,
companyId: agent.companyId,
issueId,
commentIds: extractWakeCommentIds(context),
actor: skillActor,
}));
})),
time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, {
agentId: agent.id, responsibleUserId, allowStandingDelegation: false,
})),
]);
if (mentionedSkills.status === "rejected") throw mentionedSkills.reason;
if (githubIdentity.status === "rejected") throw githubIdentity.reason;
const runScopedMentionedSkillKeys = mentionedSkills.value;
const runScopedSkillKeys =
!pinnedSkillTest && acceptedPlanContinuationWake &&
!acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation
? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY]
: runScopedMentionedSkillKeys;
const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(
db,
agent.companyId,
{
agentId: agent.id,
responsibleUserId,
allowStandingDelegation: false,
},
));
const githubSelection = githubIdentity.value;
const useHostGitHub =
!githubSelection.configured &&
trustPreset.kind === "standard" &&
@@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
);
const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = [];
const skillsPrepareStartedAtMs = Date.now();
let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs;
let skillDeliveryConfig = effectiveResolvedConfig;
const prepareSkills = async () => {
@@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
throw error;
}
};
const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills);
const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills))
.finally(() => { skillsPrepareEndedAtMs = Date.now(); });
const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([
skillsPreparation,
assignmentsPreparation,
assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))),
]);
if (preparedSkills.status === "rejected") throw preparedSkills.reason;
if (preparedAssignments.status === "rejected") throw preparedAssignments.reason;
if (preparedInstructions.status === "rejected") throw preparedInstructions.reason;
const runtimeSkillEntries = preparedSkills.value;
nativeRunnerPreparationSpans.push({
name: "skills.prepare",
parentName: "task.prepare",
startedAtMs: skillsPrepareStartedAtMs,
endedAtMs: Date.now(),
endedAtMs: skillsPrepareEndedAtMs,
});
const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
const connectorAssignments = preparedAssignments.value;
const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments));
// Both CLI adapters and native context materialization use the same resolved set.
runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills);
const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType));
// Always replace this runtime-only field; caller wake data cannot supply skills.
context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({
resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }),
let runtimeConfig = await prepareConnectionInstructionDelivery({
resolve: async () => preparedInstructions.value,
context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner",
}));
});
const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined)
? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id))
: undefined;
+49 -22
View File
@@ -1,3 +1,9 @@
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -221,7 +227,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
resumeExecutionWaitComments: () => Promise<unknown>;
resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
@@ -1267,6 +1273,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}
async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) {
if (!claimedRun || !issueId || !ownsIssue) return;
await notifyStatusCardIssueChange(tx, run.companyId);
await tx.update(issues).set({
executionRunId: claimedRun.id,
executionAgentNameKey: normalizeAgentNameKey(agent.name),
@@ -1850,13 +1857,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
return cancelled;
}
async function resumeQueuedRuns() {
async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
if ((await getSchedulingSuppression()).suppressed) return;
await resumeExecutionWaitComments();
await resumeExecutionWaitComments(scope);
const cutoff = await getWorktreeExecutionCutoff();
const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
.from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(companies.status, "active"),
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
@@ -1867,6 +1876,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
));
await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
});
}
@@ -1880,7 +1890,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
isNull(issues.executionRunId),
or(and(
sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
@@ -1919,7 +1931,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
scope.onError?.(err);
logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
});
continue;
@@ -1931,6 +1944,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
latest.errorCode === "execution_reconciliation_required";
if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
});
}
@@ -1938,6 +1952,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
}).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
});
}
@@ -1955,6 +1970,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
))
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
.where(and(
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(heartbeatRuns.status, "cancelled"),
eq(heartbeatRuns.runtimeMode, "legacy"),
@@ -1963,16 +1980,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
));
for (const run of interruptedQueues) {
await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
scope.onError?.(err);
logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
});
}
}
async function resumeQueuedRuns() {
await resumeDeferredExecutionReceipts();
await resumeQueuedRunsAfterDrain();
}
// Release admission promptly without first scanning unrelated recovery work.
// Every run still passes the ordinary per-agent claim and dispatch gates.
async function resumeQueuedRunsAfterDrain() {
async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) {
if ((await getSchedulingSuppression()).suppressed) return;
const cutoff = await getWorktreeExecutionCutoff();
const queuedRuns = await db
@@ -1982,6 +2004,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
.where(
and(
eq(heartbeatRuns.status, "queued"),
scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
eq(companies.status, "active"),
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
),
@@ -2504,17 +2527,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
// its one durable receipt ID on a skipped wake.
if (caseRequest) return { created: false };
if (executionWaitRequestId) {
await db.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
? "This task is paused. Resume it to send your saved message."
: "Waiting for task execution to be enabled. Your message is saved."),
})}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
? "This task is paused. Resume it to send your saved message."
: "Waiting for task execution to be enabled. Your message is saved."),
});
return { created: false };
}
const request = {
@@ -3402,15 +3419,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
if (executionWaitRequestId) {
await tx.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
updatedAt: new Date(),
}).where(eq(agentWakeupRequests.id, executionWaitRequestId));
await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
requestId: executionWaitRequestId }, condition, { replace: true });
return { kind: "deferred" as const, admitted: true };
}
if (durableRequest || wakeCommentId ||
hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
await tx.insert(agentWakeupRequests).values({
...durableReceiptFields,
companyId: agent.companyId, agentId, source, triggerDetail, reason,
@@ -3758,6 +3774,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
.from(agents)
.where(eq(agents.id, legacyRun.agentId))
.then((rows) => rows[0] ?? null);
await notifyStatusCardIssueChange(tx, issue.companyId);
await tx
.update(issues)
.set({
@@ -3919,6 +3936,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
if (issue.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: issue.companyId, issueId: issue.id });
await notifyRoutineExecutionStatus(tx, issue.id, issue.companyId);
await tx
.update(issues)
.set({
@@ -4028,6 +4046,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}));
}
if (followUp) {
if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
const assignmentId = followUp.assignmentId;
const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
wake.payload?.[followUpPayloadKey] === assignmentId &&
@@ -4632,6 +4651,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}
const queueOutcome = await db.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: agent.companyId });
await tx.execute(
sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`,
);
@@ -4765,6 +4785,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
companyId?: string;
limit?: number;
staleClaimMs?: number;
requestId?: string;
onDeferredDeadline?: (at: number) => void;
onError?: (error: unknown) => void;
} = {},
) {
const now = new Date();
@@ -4777,6 +4800,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
input.companyId
? eq(agentWakeupRequests.companyId, input.companyId)
: undefined,
input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
eq(agentWakeupRequests.requestedByActorType, "system"),
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
inArray(agentWakeupRequests.status, ["queued", "claimed"]),
@@ -4853,6 +4877,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
candidate.claimedAt &&
now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
) {
input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
deferred += 1;
continue;
}
@@ -5027,6 +5052,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (wakeRun) dispatched += 1;
else deferred += 1;
} catch (error) {
input.onError?.(error);
await db
.update(agentWakeupRequests)
.set({
@@ -5070,6 +5096,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
withChatControlRecoveryGate,
trackWakeup,
resumeQueuedRuns,
resumeDeferredExecutionReceipts,
resumeQueuedRunsAfterDrain,
parseHeartbeatPolicy,
nextTimerHeartbeatAt,
+2 -2
View File
@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
let settledProcessesDiscovered = false;
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
const now = new Date();
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
});
if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
await dispatchPendingNativeStatusWakeups().catch((error) => {
if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
logger.warn(
{ err: error },
"failed to dispatch persisted native status wake intents before orphan reaping",
Loaded 100 of 230 files, more files were not shown because too many files have changed in this diff. Show more