mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-02 16:58:52 +01:00
merge: preserve routine projections across retained execution producers
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
40bd4df17b
201 files changed
+8576
-1103
No files matched your search
@@ -597,8 +597,10 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => {
|
||||
await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: true, intervalSec: 600 } } });
|
||||
await vi.advanceTimersByTimeAsync(2000);
|
||||
|
||||
|
||||
expect(idleWorkSnapshot().active).toBe(1);
|
||||
// The producer committed: retain its resume deadline without keeping the
|
||||
// instance busy throughout the drain. Uncertain writes still hold admission.
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now());
|
||||
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction");
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
@@ -608,6 +610,11 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => {
|
||||
stopTaskDrain();
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(now.getTime() + 600_000), { timeout: 3000 });
|
||||
await vi.advanceTimersByTimeAsync(coordinator!.nextWakeAt()! - Date.now());
|
||||
await vi.waitFor(() => expect(f.enqueueWakeup).toHaveBeenCalledOnce());
|
||||
expect(f.enqueueWakeup).toHaveBeenCalledWith(f.agent.id, expect.objectContaining({ source: "timer", reason: "heartbeat_timer" }));
|
||||
await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: false } } });
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("does not inspect timer rows during startup drain, then reconstructs them on resume", async () => {
|
||||
|
||||
@@ -0,0 +1,897 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql, type SQL } from "drizzle-orm";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox,
|
||||
decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions,
|
||||
issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db";
|
||||
import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { writeExecutionControl } from "../services/execution-control-notifications.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { attentionService } from "../services/attention.js";
|
||||
import { companyService } from "../services/companies.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js";
|
||||
import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js";
|
||||
import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../services/recovery/origins.js";
|
||||
import { issueThreadInteractionService } from "../services/issue-thread-interactions.js";
|
||||
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
|
||||
import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js";
|
||||
import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "../services/retention-work-notifications.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL");
|
||||
const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
|
||||
(support.supported ? describe : describe.skip)("remaining attention retention", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let companyId: string, agentId: string, draining = false;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
const errors: unknown[] = [];
|
||||
const statements: string[] = [];
|
||||
const parameterCounts: number[] = [];
|
||||
beforeAll(async () => {
|
||||
if (externalUrl) await applyPendingMigrations(externalUrl);
|
||||
else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-");
|
||||
db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
db.$client.options.onnotice = () => {};
|
||||
db.$client.options.debug = (_connection, query, parameters) => { statements.push(query); parameterCounts.push(parameters.length); };
|
||||
}, 30_000);
|
||||
beforeEach(async () => {
|
||||
companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0;
|
||||
await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `A${companyId.slice(0, 6)}` });
|
||||
await db.insert(authUsers).values({ id: "board", name: "Board", email: "board@retention.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "owner", status: "active" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Origin", role: "engineer", status: "active" });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true });
|
||||
vi.setSystemTime(new Date("2026-11-01T12:00:00Z"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local client_min_messages = warning`);
|
||||
await tx.execute(sql`drop trigger if exists attention_retention_fail on decision_retention`);
|
||||
await tx.execute(sql`drop function if exists attention_retention_fail()`);
|
||||
await tx.execute(sql`truncate companies cascade`);
|
||||
});
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
const old = () => new Date(Date.now() - delay - 600_000);
|
||||
async function insertIssue(status = "todo", values: Partial<typeof issues.$inferInsert> = {}, database: Db = db) {
|
||||
const [row] = await database.insert(issues).values({ companyId, title: randomUUID(), status,
|
||||
createdByAgentId: agentId, createdAt: old(), updatedAt: old(), ...values }).returning();
|
||||
return row;
|
||||
}
|
||||
async function change<T>(write: (tx: Db) => PromiseLike<T>, scope = { companyId }) {
|
||||
return db.transaction(async tx => { await notifyAttentionRetentionWork(tx, scope); return write(tx as unknown as Db); });
|
||||
}
|
||||
const state = (kind: string, id: string) => db.select().from(decisionRetention).where(and(
|
||||
eq(decisionRetention.companyId, companyId), eq(decisionRetention.sourceKind, kind), eq(decisionRetention.sourceId, id)))
|
||||
.then(rows => rows[0] ?? null);
|
||||
async function settle() {
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 15_000 });
|
||||
}
|
||||
async function quiet() {
|
||||
await settle(); expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction");
|
||||
statements.length = 0;
|
||||
await vi.advanceTimersByTimeAsync(300_000);
|
||||
expect(statements).toEqual([]);
|
||||
expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled();
|
||||
select.mockRestore(); execute.mockRestore(); transaction.mockRestore();
|
||||
}
|
||||
async function start(service = attentionRetentionService(db)) {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) });
|
||||
await registerAttentionRetentionWork(coordinator, service, () => !draining).ready;
|
||||
return service;
|
||||
}
|
||||
async function archived(kind: string, id: string) {
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect((await state(kind, id))?.archivedAt).toBeInstanceOf(Date); }, { timeout: 15_000 });
|
||||
}
|
||||
async function loadAll(service = attentionRetentionService(db)) {
|
||||
const entries = [];
|
||||
for (;;) { const page = await service.load({ companyId }); entries.push(...page.rows ?? []); if (page.complete) return entries; }
|
||||
}
|
||||
const mutation = (sourceKind: "review", sourceId: string) => ({ companyId, sourceKind, sourceId,
|
||||
authActor: { type: "board" as const, source: "local_implicit" as const, userId: "board", companyIds: [companyId], isInstanceAdmin: true },
|
||||
actor: { actorType: "user" as const, actorId: "board", userId: "board", agentId: null, runId: null, agentApiKeyId: null, responsibleUserId: "board" } });
|
||||
|
||||
it("records the aggregate baseline SQL across old ticks and eliminates it after settlement", async () => {
|
||||
await insertIssue("done");
|
||||
const ticks: number[] = [];
|
||||
for (let tick = 0; tick < 3; tick++) {
|
||||
statements.length = 0;
|
||||
const feed = await attentionService(db).list(companyId, { includeDismissed: true, excludeSources: ["approval", "join_request"], all: true, allowUnscopedAll: true });
|
||||
await decisionRetentionService(db).autoArchive({ companyId, items: feed.items });
|
||||
ticks.push(statements.length);
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
}
|
||||
expect(ticks.every(count => count > 0)).toBe(true);
|
||||
process.stdout.write(JSON.stringify({ baselineStatementsPer30sTick: ticks, treatmentIdle300s: 0 }) + "\n");
|
||||
await start(); await quiet();
|
||||
});
|
||||
|
||||
it("has no recurring SQL when empty or when terminal source history remains", async () => {
|
||||
await insertIssue("done"); await insertIssue("cancelled");
|
||||
await start(); await quiet();
|
||||
});
|
||||
|
||||
it("wakes from the ordinary task producer, waits until due, archives with an outbox, and settles", async () => {
|
||||
await start(); await quiet();
|
||||
const row = await issueService(db).create(companyId, { title: "Review a delivered result", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId });
|
||||
await settle();
|
||||
const [saved] = await db.select({ at: sql<number>`ceil(extract(epoch from ${issues.updatedAt}) * 1000)`.mapWith(Number) }).from(issues).where(eq(issues.id, row.id));
|
||||
const due = saved.at + delay;
|
||||
expect(coordinator!.nextWakeAt()).toBe(due); expect(await state("review", row.id)).toBeNull();
|
||||
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute");
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now() - 1000);
|
||||
expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); select.mockRestore(); execute.mockRestore();
|
||||
await vi.advanceTimersByTimeAsync(2000); await archived("review", row.id); await quiet();
|
||||
const outbox = await db.select().from(decisionArchiveNotificationOutbox);
|
||||
expect(outbox).toHaveLength(1); expect(outbox[0]).toMatchObject({ companyId, sourceKind: "review", sourceId: row.id, originAgentId: agentId, originIssueId: row.id });
|
||||
expect((await db.select().from(issues).where(eq(issues.id, row.id)))[0].status).toBe("in_review");
|
||||
await coordinator!.stop(); coordinator = undefined; await start(); await quiet();
|
||||
});
|
||||
|
||||
it("registers the ordinary createChild blocker path inside its relation transaction", async () => {
|
||||
const parent = await issueService(db).create(companyId, { title: "Parent", status: "in_progress", assigneeAgentId: agentId, createdByAgentId: agentId });
|
||||
await start(); await quiet();
|
||||
const { issue: child, parentBlockerAdded } = await issueService(db).createChild(parent.id, {
|
||||
title: "Child needs review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId,
|
||||
blockParentUntilDone: true,
|
||||
});
|
||||
expect(parentBlockerAdded).toBe(true);
|
||||
expect(await db.select().from(issueRelations).where(and(eq(issueRelations.issueId, child.id),
|
||||
eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1);
|
||||
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(delay + 2000); await archived("review", child.id); await quiet();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, child.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("batches a full page of maintained reviews and confirmations by company", async () => {
|
||||
const rows = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "in_review", assigneeAgentId: agentId, updatedAt: old() }))).returning();
|
||||
await db.insert(issueThreadInteractions).values(rows.map(row => ({ companyId, issueId: row.id,
|
||||
kind: "request_confirmation", addresseeAgentId: agentId, payload: { version: 1, prompt: "Agent verdict" } })));
|
||||
statements.length = 0;
|
||||
const page = await attentionRetentionService(db).load({ companyId, sourceKind: "review" });
|
||||
expect(page.rows).toEqual([]); expect(page.complete).toBe(false);
|
||||
expect(statements.length).toBeLessThanOrEqual(20);
|
||||
expect(statements.filter(query => query.includes('from "agents"')).length).toBeLessThanOrEqual(1);
|
||||
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId));
|
||||
statements.length = 0;
|
||||
const interactions = await attentionRetentionService(db).load({ companyId, sourceKind: "issue_thread_interaction" });
|
||||
expect(interactions.rows).toHaveLength(100); expect(interactions.complete).toBe(false);
|
||||
expect(statements.length).toBeLessThanOrEqual(6);
|
||||
expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("batches decision and recovery source links and rechecks a late hidden origin", async () => {
|
||||
const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "todo", updatedAt: old() }))).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded" }).returning();
|
||||
const decisionRows = await db.insert(decisions).values(origins.map(origin => ({ companyId,
|
||||
originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, title: "Decision", body: "Choose",
|
||||
options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }))).returning();
|
||||
const recoveryRows = await db.insert(issueRecoveryActions).values(origins.map(origin => ({ companyId,
|
||||
sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture",
|
||||
nextAction: "Choose", updatedAt: old() }))).returning();
|
||||
const service = attentionRetentionService(db);
|
||||
const pages = [];
|
||||
for (const sourceKind of ["decision", "recovery_action"] as const) {
|
||||
statements.length = 0;
|
||||
const page = await service.load({ companyId, sourceKind });
|
||||
expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false);
|
||||
expect(statements.length).toBeLessThanOrEqual(3);
|
||||
pages.push(page.rows!);
|
||||
}
|
||||
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[99].id));
|
||||
const hiddenDecision = pages[0].find(entry => entry.sourceId === decisionRows[99].id)!;
|
||||
const hiddenRecovery = pages[1].find(entry => entry.sourceId === recoveryRows[99].id)!;
|
||||
expect(await service.check(hiddenDecision)).toBeNull(); expect(await service.check(hiddenRecovery)).toBeNull();
|
||||
expect(await state("decision", hiddenDecision.sourceId)).toBeNull(); expect(await state("recovery_action", hiddenRecovery.sourceId)).toBeNull();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("batches 100 failed runs and rechecks late visibility and supersession before archive", async () => {
|
||||
const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "todo", updatedAt: old() }))).returning();
|
||||
const runs = await db.insert(heartbeatRuns).values(origins.map(origin => ({ companyId, agentId,
|
||||
status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }))).returning();
|
||||
await db.insert(heartbeatRunEvents).values(runs.map(run => ({ companyId, agentId, runId: run.id,
|
||||
seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: batch fixture" })));
|
||||
const service = attentionRetentionService(db), scope = { companyId, sourceKind: "failed_run" as const };
|
||||
statements.length = 0;
|
||||
const page = await service.load(scope);
|
||||
expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false);
|
||||
expect(statements.length).toBeLessThanOrEqual(6);
|
||||
const hiddenRun = runs[98], supersededRun = runs[99];
|
||||
const hidden = page.rows!.find(entry => entry.sourceId === hiddenRun.id)!;
|
||||
const superseded = page.rows!.find(entry => entry.sourceId === supersededRun.id)!;
|
||||
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[98].id));
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded",
|
||||
contextSnapshot: { issueId: origins[99].id }, createdAt: new Date(old().getTime() + 60_000), finishedAt: old() });
|
||||
expect(await service.check(hidden)).toBeNull(); expect(await service.check(superseded)).toBeNull();
|
||||
expect(await state("failed_run", hiddenRun.id)).toBeNull(); expect(await state("failed_run", supersededRun.id)).toBeNull();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
expect((await service.load(scope)).complete).toBe(true);
|
||||
const refreshed = await service.load(scope);
|
||||
expect(refreshed.rows).toHaveLength(98);
|
||||
expect(refreshed.rows!.some(entry => [hiddenRun.id, supersededRun.id].includes(entry.sourceId))).toBe(false);
|
||||
});
|
||||
|
||||
it("batches 100 blocker graphs with exact terminal timestamps and fresh terminal visibility", async () => {
|
||||
const roots = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "blocked", updatedAt: old(), blockedTransitionAt: old(),
|
||||
unblockDescriptor: { owner: "board", action: "Pick an owner" } }))).returning();
|
||||
const terminals = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "backlog", assigneeAgentId: agentId, updatedAt: old() }))).returning();
|
||||
await db.insert(issueRelations).values(roots.map((root, index) => ({ companyId,
|
||||
issueId: terminals[index].id, relatedIssueId: root.id, type: "blocks" })));
|
||||
await db.execute(sql`update issues set updated_at = updated_at + interval '0.000321 seconds' where id = ${terminals[99].id}::uuid`);
|
||||
const service = attentionRetentionService(db), scope = { companyId, sourceKind: "blocker_attention" as const };
|
||||
statements.length = 0;
|
||||
const page = await service.load(scope);
|
||||
const queries = [...statements];
|
||||
expect(page.rows).toHaveLength(200); expect(page.complete).toBe(false);
|
||||
const retentionReads = queries.filter(query => query.includes('from "decision_retention"'));
|
||||
expect(retentionReads).toHaveLength(200);
|
||||
expect(queries.length - retentionReads.length).toBeLessThanOrEqual(30);
|
||||
const terminal = page.rows!.find(entry => entry.rootId === roots[99].id && entry.variant === "terminal")!;
|
||||
expect(terminal.sourceId).toBe(terminals[99].id);
|
||||
const [expected] = await db.execute(sql`select updated_at::text as at,
|
||||
ceil(extract(epoch from updated_at) * 1000 + ${delay}) as due from issues where id = ${terminals[99].id}::uuid`);
|
||||
expect(terminal.activityAt).toBe(expected.at); expect(terminal.dueAt).toBe(Number(expected.due));
|
||||
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, terminals[99].id));
|
||||
expect(await service.check(terminal)).toBeNull();
|
||||
expect(await state("blocker_attention", terminals[99].id)).toBeNull();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
expect((await service.load(scope)).complete).toBe(true);
|
||||
const refreshed = await service.load(scope);
|
||||
expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["depth", "nodes"] as const)("preserves shallow terminal selection when an unrelated graph hits its %s limit", async limit => {
|
||||
const root = await insertIssue("blocked", { updatedAt: new Date() });
|
||||
const middle = await insertIssue("todo", { updatedAt: new Date() });
|
||||
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId });
|
||||
const heavy = await insertIssue("blocked", { updatedAt: new Date() });
|
||||
const count = limit === "depth" ? BLOCKER_ATTENTION_MAX_DEPTH + 1 : BLOCKER_ATTENTION_MAX_NODES;
|
||||
const heavyNodes = await db.insert(issues).values(Array.from({ length: count }, () => ({ companyId,
|
||||
title: randomUUID(), status: "todo", updatedAt: new Date() }))).returning();
|
||||
await db.insert(issueRelations).values([
|
||||
{ companyId, issueId: middle.id, relatedIssueId: root.id, type: "blocks" },
|
||||
{ companyId, issueId: terminal.id, relatedIssueId: middle.id, type: "blocks" },
|
||||
...heavyNodes.map((node, index) => ({ companyId, issueId: node.id,
|
||||
relatedIssueId: limit === "nodes" || index === 0 ? heavy.id : heavyNodes[index - 1].id, type: "blocks" })),
|
||||
]);
|
||||
const owner = issueService(db);
|
||||
const expected = (await owner.listBlockerAttention(companyId, [root])).get(root.id)!;
|
||||
const limited = (await owner.listBlockerAttention(companyId, [heavy])).get(heavy.id)!;
|
||||
expect(expected.terminalBlockerIssueId).toBe(terminal.id);
|
||||
expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId);
|
||||
for (const input of [[root, heavy], [heavy, root]]) {
|
||||
const batch = await owner.listBlockerAttentionBatch(companyId, input);
|
||||
if (!batch.deferredRootIds.has(root.id)) expect(batch.attention.get(root.id)).toEqual(expected);
|
||||
if (!batch.deferredRootIds.has(heavy.id)) expect(batch.attention.get(heavy.id)).toEqual(limited);
|
||||
expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
|
||||
if (limit === "depth") expect(batch.deferredRootIds.has(root.id)).toBe(false);
|
||||
}
|
||||
const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" });
|
||||
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
|
||||
expect(entry.dueAt).toBeLessThanOrEqual(Date.now());
|
||||
if (limit === "depth") expect(entry.sourceId).toBe(terminal.id);
|
||||
await attentionRetentionService(db).check(entry);
|
||||
expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps root-specific reachability and cycle budgets when another root explores farther", async () => {
|
||||
const root = await insertIssue("blocked");
|
||||
const chain = await db.insert(issues).values(Array.from({ length: BLOCKER_ATTENTION_MAX_DEPTH + 1 }, (_, index) => ({
|
||||
companyId, title: randomUUID(), status: index === BLOCKER_ATTENTION_MAX_DEPTH ? "in_progress"
|
||||
: index === BLOCKER_ATTENTION_MAX_DEPTH - 2 ? "blocked" : "todo", updatedAt: old() }))).returning();
|
||||
const overlapping = chain[BLOCKER_ATTENTION_MAX_DEPTH - 2];
|
||||
const cycle = await insertIssue("blocked");
|
||||
const leaf = await insertIssue("backlog", { assigneeAgentId: agentId });
|
||||
await db.insert(issueRelations).values([
|
||||
...chain.map((node, index) => ({ companyId, issueId: node.id, relatedIssueId: index ? chain[index - 1].id : root.id, type: "blocks" })),
|
||||
{ companyId, issueId: leaf.id, relatedIssueId: cycle.id, type: "blocks" },
|
||||
{ companyId, issueId: cycle.id, relatedIssueId: leaf.id, type: "blocks" },
|
||||
]);
|
||||
const owner = issueService(db), roots = [root, overlapping, cycle];
|
||||
const singles = new Map();
|
||||
for (const item of roots) singles.set(item.id, (await owner.listBlockerAttention(companyId, [item])).get(item.id));
|
||||
expect(singles.get(root.id).blockingTreeLive).toBe(false);
|
||||
expect(singles.get(overlapping.id).blockingTreeLive).toBe(true);
|
||||
for (const input of [roots, [...roots].reverse()]) {
|
||||
const batch = await owner.listBlockerAttention(companyId, input);
|
||||
for (const item of roots) expect(batch.get(item.id)).toEqual(singles.get(item.id));
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds every graph query when independent root trees contain over 65535 unfinished nodes", async () => {
|
||||
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 70) n`);
|
||||
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-branch-' || n, 'todo', ${old().toISOString()}::timestamptz
|
||||
from issues root cross join generate_series(1, 10) n
|
||||
where root.company_id = ${companyId}::uuid and root.status = 'blocked'`);
|
||||
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, branch.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from issues branch cross join generate_series(1, 99) n
|
||||
where branch.company_id = ${companyId}::uuid and branch.title like 'parameter-branch-%'`);
|
||||
const completed = await insertIssue("blocked");
|
||||
const shallow = await insertIssue("blocked", { updatedAt: new Date() });
|
||||
const middle = await insertIssue("todo", { updatedAt: new Date() });
|
||||
const overdue = await insertIssue("backlog", { assigneeAgentId: agentId });
|
||||
await db.insert(issueRelations).values([
|
||||
{ companyId, issueId: middle.id, relatedIssueId: shallow.id, type: "blocks" },
|
||||
{ companyId, issueId: overdue.id, relatedIssueId: middle.id, type: "blocks" },
|
||||
]);
|
||||
const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked")));
|
||||
parameterCounts.length = 0;
|
||||
const result = await issueService(db).listBlockerAttentionBatch(companyId, roots).catch(error => {
|
||||
throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`);
|
||||
});
|
||||
expect(result.attention.size).toBe(72);
|
||||
expect(result.deferredRootIds.has(completed.id)).toBe(false);
|
||||
expect(result.attention.get(completed.id)?.terminalBlockerIssueId).toBe(completed.id);
|
||||
expect([...result.attention.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true);
|
||||
expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100);
|
||||
expect(parameterCounts.length).toBeLessThanOrEqual(100);
|
||||
expect(result.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
|
||||
expect(result.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2);
|
||||
expect(result.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6);
|
||||
expect(result.deferredRootIds.has(shallow.id)).toBe(true);
|
||||
const service = attentionRetentionService(db);
|
||||
const page = await service.load({ companyId, sourceKind: "blocker_attention" });
|
||||
const immediate = page.rows!.find(entry => entry.rootId === shallow.id && entry.variant === "terminal")!;
|
||||
expect(immediate.dueAt).toBeLessThanOrEqual(Date.now());
|
||||
await service.check(immediate);
|
||||
expect((await state("blocker_attention", overdue.id))?.archivedAt).toBeInstanceOf(Date);
|
||||
}, 120_000);
|
||||
|
||||
it("preserves observed live work when a single large root exhausts shared fetch capacity", async () => {
|
||||
const root = await insertIssue("blocked");
|
||||
const live = await insertIssue("in_progress", { assigneeAgentId: agentId });
|
||||
await db.insert(issueRelations).values({ companyId, issueId: live.id, relatedIssueId: root.id, type: "blocks" });
|
||||
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, ${root.id}::uuid, 'bounded-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 3000) n`);
|
||||
const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]);
|
||||
expect(batch.deferredRootIds.has(root.id)).toBe(true);
|
||||
expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true);
|
||||
const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" });
|
||||
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
|
||||
expect(await service.check(entry)).toBeNull();
|
||||
expect(await state("blocker_attention", root.id)).toBeNull();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(["in_progress", "run", "confirmation"])("preserves observed child live evidence (%s) after explicit blockers fill the node budget", async (kind) => {
|
||||
const root = await insertIssue("blocked");
|
||||
await db.execute(sql`with blockers as (
|
||||
insert into issues (id, company_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'explicit-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 1999) n returning id)
|
||||
insert into issue_relations (company_id, issue_id, related_issue_id, type)
|
||||
select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`);
|
||||
const child = await insertIssue(kind === "in_progress" ? "in_progress" : "todo", { parentId: root.id, assigneeAgentId: agentId });
|
||||
if (kind === "run") {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
|
||||
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, child.id));
|
||||
} else if (kind === "confirmation") {
|
||||
await db.insert(issueThreadInteractions).values({ companyId, issueId: child.id, kind: "request_confirmation", status: "pending", payload: { prompt: "Approve?" } });
|
||||
}
|
||||
parameterCounts.length = 0;
|
||||
const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]);
|
||||
expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
|
||||
expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2);
|
||||
expect(batch.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6);
|
||||
expect(parameterCounts.length).toBeLessThanOrEqual(100);
|
||||
expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100);
|
||||
expect(batch.deferredRootIds.has(root.id)).toBe(true);
|
||||
expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true);
|
||||
const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" });
|
||||
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
|
||||
expect(await service.check(entry)).toBeNull();
|
||||
expect(await db.select().from(decisionRetention)).toHaveLength(0);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => {
|
||||
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 4) n`);
|
||||
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, root.id, 'held-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from issues root cross join generate_series(1, 600) n
|
||||
where root.company_id = ${companyId}::uuid and root.status = 'blocked'`);
|
||||
await db.execute(sql`insert into decision_retention (company_id, source_kind, source_id, source_activity_at, keep)
|
||||
select company_id, 'blocker_attention', id::text, updated_at, true from issues
|
||||
where company_id = ${companyId}::uuid and status = 'backlog'`);
|
||||
const service = attentionRetentionService(db), check = vi.spyOn(service, "check");
|
||||
await start(service); await quiet();
|
||||
expect(check).toHaveBeenCalledTimes(4);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("bounds high-fanin ordinary issue writes to one coalescible hint", async () => {
|
||||
const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load");
|
||||
await start(service); await settle(); load.mockClear();
|
||||
const leaf = await insertIssue("in_review", { updatedAt: new Date(), assigneeUserId: "board" });
|
||||
await db.execute(sql`with parents as (
|
||||
insert into issues (id, company_id, title, status)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'fan-in-' || n, 'todo'
|
||||
from generate_series(1, 2100) n returning id)
|
||||
insert into issue_relations (company_id, issue_id, related_issue_id, type)
|
||||
select ${companyId}::uuid, ${leaf.id}::uuid, id, 'blocks' from parents`);
|
||||
const hints: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
|
||||
if (event === "settled") hints.push(...values ?? []);
|
||||
});
|
||||
let release!: () => void, entered = false;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; });
|
||||
statements.length = 0;
|
||||
try {
|
||||
await issueService(db).update(leaf.id, { title: "High fan-in ordinary edit 0" });
|
||||
await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 });
|
||||
for (let n = 1; n < 8; n++) await issueService(db).update(leaf.id, { title: `High fan-in ordinary edit ${n}` });
|
||||
} finally { release(); unsubscribe(); }
|
||||
expect(statements.filter(query => query.includes("with recursive affected"))).toHaveLength(8);
|
||||
expect(hints).toHaveLength(8);
|
||||
expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId })]));
|
||||
await settle();
|
||||
expect(load.mock.calls.length).toBeLessThanOrEqual(3);
|
||||
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true);
|
||||
let query!: Parameters<typeof db.execute>[0], returned = 0;
|
||||
await db.transaction(async tx => {
|
||||
const execute = tx.execute.bind(tx);
|
||||
vi.spyOn(tx, "execute").mockImplementation(async statement => {
|
||||
const result = await execute(statement);
|
||||
if (new PgDialect().sqlToQuery(statement as SQL).sql.includes("with recursive affected")) {
|
||||
query = statement; returned = result.length;
|
||||
}
|
||||
return result;
|
||||
});
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: leaf.id });
|
||||
});
|
||||
expect(returned).toBe(101);
|
||||
const plans = await db.execute(sql`explain (analyze, format json) ${query as SQL}`);
|
||||
const plan = (plans[0]["QUERY PLAN"] as any)[0].Plan;
|
||||
expect(plan["Node Type"]).toBe("Limit");
|
||||
expect(plan["Actual Rows"]).toBe(101);
|
||||
const nodes: any[] = [], collect = (node: any) => { nodes.push(node); for (const child of node.Plans ?? []) collect(child); };
|
||||
collect(plan);
|
||||
const recursive = nodes.filter(node => node["Node Type"] === "Recursive Union");
|
||||
const relations = nodes.filter(node => node["Relation Name"] === "issue_relations");
|
||||
expect(recursive).toHaveLength(1);
|
||||
expect(recursive[0]["Actual Rows"]).toBe(101);
|
||||
expect(relations.length).toBeGreaterThan(0);
|
||||
expect(relations.every(node => node["Actual Rows"] * node["Actual Loops"] <= 101)).toBe(true);
|
||||
process.stdout.write(JSON.stringify({ reverseFanInPlan: plan }) + "\n");
|
||||
});
|
||||
|
||||
it("retains far ancestor deadlines when a cyclic chain exceeds the precise hint budget", async () => {
|
||||
const service = attentionRetentionService(db), load = vi.spyOn(service, "load");
|
||||
await start(service); await settle(); load.mockClear();
|
||||
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at, assignee_user_id)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'chain-' || n,
|
||||
case when n = 150 then 'in_review' else 'todo' end,
|
||||
case when n = 150 then ${new Date(Date.now() - delay + 15_000).toISOString()}::timestamptz else now() end,
|
||||
case when n = 150 then 'board' else null end
|
||||
from generate_series(0, 150) n`);
|
||||
await db.execute(sql`update issues child set parent_id = parent.id from issues parent
|
||||
where child.company_id = ${companyId}::uuid and parent.company_id = child.company_id
|
||||
and parent.title = 'chain-' || ((substring(child.title from 7)::integer + 1) % 151)`);
|
||||
const [leaf, far] = await Promise.all([db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-0"))).then(rows => rows[0]),
|
||||
db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-150"))).then(rows => rows[0])]);
|
||||
await issueService(db).update(leaf.id, { title: "Ordinary deep chain edit" });
|
||||
await settle();
|
||||
expect(load.mock.calls.some(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true);
|
||||
expect(await state("review", far.id)).toBeNull();
|
||||
expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
await archived("review", far.id);
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("canonicalizes a complete cyclic affected set into one bounded batch hint", async () => {
|
||||
const a = await insertIssue(), b = await insertIssue(), c = await insertIssue();
|
||||
await db.update(issues).set({ parentId: b.id }).where(eq(issues.id, a.id));
|
||||
await db.update(issues).set({ parentId: c.id }).where(eq(issues.id, b.id));
|
||||
await db.update(issues).set({ parentId: a.id }).where(eq(issues.id, c.id));
|
||||
const hints: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
|
||||
if (event === "settled") hints.push(...values ?? []);
|
||||
});
|
||||
try {
|
||||
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: a.id }));
|
||||
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: c.id }));
|
||||
const wide = { companyId, issueIds: [b.id, a.id, c.id, a.id], ignored: {} as unknown };
|
||||
wide.ignored = wide;
|
||||
await db.transaction(tx => notifyAttentionRetentionWork(tx, wide));
|
||||
} finally { unsubscribe(); }
|
||||
expect(hints).toHaveLength(3);
|
||||
expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId, issueIds: [a.id, b.id, c.id].sort() })]));
|
||||
});
|
||||
|
||||
it("canonicalizes wide circular notification variables to the supported identity fields", async () => {
|
||||
const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = [];
|
||||
const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown };
|
||||
wide.ignored = wide;
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
|
||||
if (event === "settled") hints.push(...values ?? []);
|
||||
});
|
||||
try { await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); } finally { unsubscribe(); }
|
||||
expect(hints).toEqual([JSON.stringify({ companyId, sourceKind: "review", sourceId, issueId })]);
|
||||
});
|
||||
|
||||
it("keeps held, burst and sustained ordinary issue writes scoped while an unrelated deadline fires", async () => {
|
||||
const row = await issueService(db).create(companyId, { title: "Active review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId });
|
||||
const urgent = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date(Date.now() - delay + 10_000) });
|
||||
const otherCompany = randomUUID();
|
||||
await db.insert(companies).values({ id: otherCompany, name: "Unrelated", issuePrefix: "BURST" });
|
||||
const foreign = await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany,
|
||||
title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))).returning();
|
||||
await db.insert(decisionRetention).values(foreign.map(issue => ({ companyId: otherCompany, sourceKind: "review", sourceId: issue.id, sourceActivityAt: issue.updatedAt, keep: true })));
|
||||
const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load");
|
||||
await start(service); await settle(); load.mockClear();
|
||||
let release!: () => void, entered = false;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; });
|
||||
statements.length = 0;
|
||||
await issueService(db).update(row.id, { title: "Held mutation" });
|
||||
await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 });
|
||||
await Promise.all(Array.from({ length: 5 }, (_, index) => issueService(db).update(row.id, { title: `Burst ${index}` })));
|
||||
release(); await settle();
|
||||
for (let index = 0; index < 8; index++) {
|
||||
await issueService(db).update(row.id, { title: `Steady ${index}` }); await vi.advanceTimersByTimeAsync(1000); await settle();
|
||||
}
|
||||
await archived("review", urgent.id);
|
||||
const routine = load.mock.calls.map(([scope]) => scope).filter(scope => !scope?.sourceId);
|
||||
expect(routine.length).toBeGreaterThan(0);
|
||||
expect(routine.every(scope => scope?.companyId === companyId && scope.issueId === row.id)).toBe(true);
|
||||
const scans = statements.filter(query => query.includes("with source_candidates"));
|
||||
expect(scans.length).toBeGreaterThan(0);
|
||||
expect(scans.every(query => !query.includes("budget_incidents") && !query.includes("agent_error_alert"))).toBe(true);
|
||||
expect(await state("review", row.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("wakes all linked issue sources when an ordinary visibility mutation reveals their origin", async () => {
|
||||
const origin = await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board", responsibleUserId: "board" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }).returning();
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: visibility" });
|
||||
const [decision] = await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id,
|
||||
title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }).returning();
|
||||
const [recovery] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }).returning();
|
||||
const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }).returning();
|
||||
await start(); await quiet();
|
||||
await issueService(db).update(origin.id, { hiddenAt: null });
|
||||
for (const [kind, id] of [["decision", decision.id], ["recovery_action", recovery.id], ["failed_run", run.id], ["issue_thread_interaction", interaction.id]]) await archived(kind, id);
|
||||
await settle(); expect(await state("review", origin.id)).toBeNull(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
});
|
||||
|
||||
it("invalidates old and new parents, terminal visibility and repointed blocker relations", async () => {
|
||||
const oldParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" });
|
||||
const newParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" });
|
||||
const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" });
|
||||
const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear();
|
||||
await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle();
|
||||
const ids = load.mock.calls.flatMap(([scope]) => scope?.issueIds ?? (scope?.issueId ? [scope.issueId] : []));
|
||||
expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id]));
|
||||
load.mockClear();
|
||||
await issueService(db).update(terminal.id, { hiddenAt: null }); await settle();
|
||||
expect(await state("blocker_attention", terminal.id)).toBeNull();
|
||||
const visible = await service.load({ companyId, issueId: newParent.id });
|
||||
expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true);
|
||||
expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id || scope?.issueIds?.includes(newParent.id))).toBe(true);
|
||||
const next = await insertIssue("backlog", { assigneeAgentId: agentId });
|
||||
await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id);
|
||||
});
|
||||
|
||||
it("cancels and reactivates a blocker deadline through ordinary liveness incident creation and hiding", async () => {
|
||||
const root = await insertIssue("blocked");
|
||||
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(Date.now() - delay + 10_000) });
|
||||
await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" });
|
||||
await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
const incident = await issueService(db).create(companyId, { title: "Resolve liveness", status: "todo", assigneeUserId: "board",
|
||||
createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
|
||||
originId: buildIssueGraphLivenessIncidentKey({ companyId, issueId: root.id, state: "needs_attention", blockerIssueId: terminal.id }) });
|
||||
await quiet(); expect(await state("blocker_attention", terminal.id)).toBeNull();
|
||||
await issueService(db).update(incident.id, { hiddenAt: new Date() });
|
||||
await archived("blocker_attention", terminal.id); await quiet();
|
||||
});
|
||||
|
||||
it("resolves opaque run issue references through trusted company issue rows", async () => {
|
||||
const issue = await insertIssue();
|
||||
const [valid, opaque] = await db.insert(heartbeatRuns).values([
|
||||
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: issue.id } },
|
||||
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: "none" } },
|
||||
]).returning();
|
||||
const hints: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
|
||||
if (event === "settled") hints.push(...values ?? []);
|
||||
});
|
||||
try {
|
||||
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: valid.id }));
|
||||
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: opaque.id }));
|
||||
} finally { unsubscribe(); }
|
||||
expect(hints).toEqual([JSON.stringify({ companyId, issueId: issue.id }), JSON.stringify({ companyId })]);
|
||||
});
|
||||
|
||||
it("accepts ordinary liveness incident creation and completion with a none leaf sentinel", async () => {
|
||||
const source = await insertIssue("blocked", { responsibleUserId: "board" });
|
||||
const service = issueService(db);
|
||||
const incident = await service.create(companyId, { title: "Invalid participant", status: "todo", assigneeUserId: "board",
|
||||
createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
|
||||
originId: `harness_liveness:${companyId}:${source.id}:invalid_review_participant:none` });
|
||||
await service.update(source.id, { blockedByIssueIds: [incident.id] });
|
||||
await service.update(incident.id, { status: "done" });
|
||||
expect((await service.getById(incident.id))?.status).toBe("done");
|
||||
expect((await service.getRelationSummaries(source.id)).blockedBy).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["hidden", "done"] as const)("reactivates an older review when moved stranded recovery is %s", async transition => {
|
||||
const review = await insertIssue("in_review", { assigneeAgentId: agentId, responsibleUserId: "board" });
|
||||
const parent = await insertIssue("todo", { responsibleUserId: "board" });
|
||||
const recovery = await insertIssue("todo", { parentId: review.id, originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,
|
||||
originId: review.id, assigneeUserId: "board", responsibleUserId: "board" });
|
||||
await issueService(db).update(recovery.id, { parentId: parent.id });
|
||||
expect((await issueService(db).listReviewAttention(companyId, [review])).get(review.id)?.state).not.toBe("stalled");
|
||||
await start(); await quiet();
|
||||
await issueService(db).update(recovery.id, transition === "hidden" ? { hiddenAt: new Date() } : { status: "done" });
|
||||
await settle();
|
||||
expect((await state("review", review.id))?.archivedAt).toBeInstanceOf(Date);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, review.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("scopes ordinary execution-control writes to their company", async () => {
|
||||
const otherCompany = randomUUID();
|
||||
await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" });
|
||||
await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany,
|
||||
title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() })));
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
|
||||
const service = attentionRetentionService(db), load = vi.spyOn(service, "load");
|
||||
await start(service); await quiet(); load.mockClear();
|
||||
for (let count = 0; count < 3; count++) {
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization,
|
||||
tx => tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)), { companyId });
|
||||
await quiet();
|
||||
}
|
||||
expect(load).toHaveBeenCalled();
|
||||
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true);
|
||||
});
|
||||
|
||||
it("registers and cancels budget deadlines through ordinary policy and incident producers", async () => {
|
||||
const service = budgetService(db);
|
||||
await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 100,
|
||||
warnPercent: 85, hardStopEnabled: false, notifyEnabled: true }, "board");
|
||||
const retention = attentionRetentionService(db), load = vi.spyOn(retention, "load");
|
||||
await start(retention); await quiet(); load.mockClear();
|
||||
const [event] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai",
|
||||
billingType: "metered_api", model: "fixture", inputTokens: 100, outputTokens: 20,
|
||||
costCents: 90, occurredAt: new Date() }).returning();
|
||||
await service.evaluateCostEvent(event);
|
||||
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
const [incident] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId));
|
||||
expect(incident).toMatchObject({ status: "open", thresholdType: "soft" });
|
||||
expect(await state("budget_alert", incident.id)).toBeNull();
|
||||
await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 200 }, "board");
|
||||
await quiet();
|
||||
expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0].status).toBe("resolved");
|
||||
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true);
|
||||
});
|
||||
|
||||
it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => {
|
||||
const origin = await insertIssue();
|
||||
const review = await insertIssue("in_review", { assigneeUserId: "board" });
|
||||
const blocker = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Choose a dependency" }, blockedTransitionAt: old() });
|
||||
await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board" });
|
||||
const hidden = await insertIssue("todo", { hiddenAt: old() });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), updatedAt: old(), finishedAt: old() }).returning();
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: fixture" });
|
||||
await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id,
|
||||
title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() });
|
||||
await db.insert(issueThreadInteractions).values([
|
||||
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() },
|
||||
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeAgentId: agentId, updatedAt: old() },
|
||||
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeUserId: "board", updatedAt: old() },
|
||||
{ companyId, issueId: hidden.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() },
|
||||
]);
|
||||
await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() });
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId, windowKind: "calendar_month_utc", amount: 100 }).returning();
|
||||
await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: old(), windowEnd: new Date(), thresholdType: "hard", amountLimit: 100, amountObserved: 100, updatedAt: old() });
|
||||
await db.insert(agents).values({ companyId, name: "Error", role: "engineer", status: "error", updatedAt: old() });
|
||||
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true, includeDismissed: true, excludeSources: ["approval", "join_request"] });
|
||||
const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort();
|
||||
const actual = [...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort();
|
||||
expect(new Set(feed.items.map(item => item.sourceKind)).size).toBe(8); expect(actual).toEqual(expected);
|
||||
expect(actual).toContain(`review:${review.id}`); expect(actual).toContain(`blocker_attention:${blocker.id}`);
|
||||
await start(); for (const item of feed.items) await archived(item.sourceKind, item.subject.id); await quiet();
|
||||
});
|
||||
|
||||
it("keeps unseen sources, cancels their deadline, and reactivates when unkept or revived", async () => {
|
||||
const row = await insertIssue("in_review", { assigneeUserId: "board" });
|
||||
await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: true });
|
||||
await start(); await quiet(); expect((await state("review", row.id))?.archivedAt).toBeNull();
|
||||
await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: false });
|
||||
await archived("review", row.id); await quiet();
|
||||
await decisionRetentionService(db).revive(mutation("review", row.id)); await archived("review", row.id); await quiet();
|
||||
});
|
||||
|
||||
it("moves deadlines on source activity and cancels when the source leaves attention", async () => {
|
||||
const row = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date() });
|
||||
await start(); await settle(); const first = coordinator!.nextWakeAt()!;
|
||||
await change(tx => tx.update(issues).set({ updatedAt: new Date(Date.now() - delay + 600_000) }).where(eq(issues.id, row.id)));
|
||||
await settle(); expect(coordinator!.nextWakeAt()).toBeLessThan(first);
|
||||
await issueService(db).update(row.id, { status: "todo" }); await quiet(); expect(await state("review", row.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves outer transaction ownership and first writes during drain", async () => {
|
||||
await start(); await quiet(); draining = true;
|
||||
let id: string | undefined;
|
||||
await db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
await notifyAttentionRetentionWork(nested, { companyId });
|
||||
id = (await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db)).id;
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(60_000); expect(await state("review", id!)).toBeNull();
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(120_000); expect(await state("review", id!)).toBeNull();
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", id!); await quiet();
|
||||
await expect(db.transaction(async outer => {
|
||||
await outer.transaction(async nested => { await notifyAttentionRetentionWork(nested, { companyId }); await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db); });
|
||||
throw new Error("rollback");
|
||||
})).rejects.toThrow("rollback");
|
||||
await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("isolates companies, catches up at activation, and bounds retained-history pages", async () => {
|
||||
const other = randomUUID(); await db.insert(companies).values({ id: other, name: "Paused", issuePrefix: "RP", status: "paused" });
|
||||
const row = await insertIssue("in_review", { companyId: other, assigneeUserId: "board" });
|
||||
await db.insert(issues).values(Array.from({ length: 1000 }, () => ({ companyId, title: randomUUID(), status: "done", updatedAt: old() })));
|
||||
const service = attentionRetentionService(db); const page = await service.load({ companyId });
|
||||
expect(page).toEqual({ rows: [], complete: true });
|
||||
await start(service); await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(0);
|
||||
await companyService(db).update(other, { status: "active" });
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(1); });
|
||||
expect((await db.select().from(decisionRetention))[0]).toMatchObject({ companyId: other, sourceId: row.id }); await quiet();
|
||||
});
|
||||
it("preserves terminal blocker source identity and its own activity deadline", async () => {
|
||||
const root = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Pick an owner" }, blockedTransitionAt: old() });
|
||||
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(old().getTime() + 60_000) });
|
||||
await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" });
|
||||
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
|
||||
const entries = await loadAll();
|
||||
const expected = feed.items.filter(item => item.sourceKind === "blocker_attention")
|
||||
.map(item => `${item.subject.id}:${new Date(item.activityAt).toISOString()}`).sort();
|
||||
expect(entries.filter(item => item.sourceKind === "blocker_attention")
|
||||
.map(item => `${item.sourceId}:${new Date(item.activityAt).toISOString()}`).sort()).toEqual(expected);
|
||||
expect(expected).toContain(`${terminal.id}:${terminal.updatedAt.toISOString()}`);
|
||||
await start(); await archived("blocker_attention", terminal.id); await archived("blocker_attention", root.id); await quiet();
|
||||
expect((await state("blocker_attention", terminal.id))?.sourceActivityAt).toEqual(terminal.updatedAt);
|
||||
expect((await state("blocker_attention", root.id))?.sourceActivityAt).toEqual(root.blockedTransitionAt);
|
||||
});
|
||||
|
||||
it("matches newest confirmations, independent gates, org invokability, and conversation visibility", async () => {
|
||||
const origin = await insertIssue();
|
||||
const conversation = await insertIssue("in_review", { conversationAgentId: agentId, conversationUserId: "board", conversationState: "waiting", assigneeAgentId: agentId });
|
||||
const hidden = await insertIssue("todo", { hiddenAt: old() });
|
||||
const [paused, manager, unhealthy] = await db.insert(agents).values([
|
||||
{ companyId, name: "Paused", role: "engineer", status: "paused" },
|
||||
{ companyId, name: "Terminated", role: "manager", status: "terminated" },
|
||||
{ companyId, name: "Unhealthy", role: "engineer", status: "active" },
|
||||
]).returning();
|
||||
await db.update(agents).set({ reportsTo: manager.id }).where(eq(agents.id, unhealthy.id));
|
||||
const common = { companyId, issueId: origin.id, kind: "request_confirmation", payload: { prompt: "Approve?" }, createdAt: old(), updatedAt: old() };
|
||||
const [older, newest, independent] = await db.insert(issueThreadInteractions).values([
|
||||
common, { ...common, createdAt: new Date(old().getTime() + 1000) },
|
||||
{ ...common, payload: { prompt: "Independent", qualityReview: { artifactKey: "copy" } } },
|
||||
{ ...common, createdAt: new Date(), addresseeUserId: "board" },
|
||||
{ ...common, addresseeAgentId: agentId },
|
||||
{ ...common, issueId: hidden.id },
|
||||
{ ...common, issueId: conversation.id, kind: "ask_user_questions", payload: { questions: [] } },
|
||||
{ ...common, issueId: conversation.id, addresseeAgentId: paused.id },
|
||||
{ ...common, issueId: conversation.id, addresseeAgentId: unhealthy.id, kind: "ask_user_questions", payload: { questions: [] } },
|
||||
] as typeof issueThreadInteractions.$inferInsert[]).returning();
|
||||
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
|
||||
const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort();
|
||||
expect([...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort()).toEqual(expected);
|
||||
expect(expected).not.toContain(`issue_thread_interaction:${older.id}`);
|
||||
expect(expected).toContain(`issue_thread_interaction:${newest.id}`); expect(expected).toContain(`issue_thread_interaction:${independent.id}`);
|
||||
await start(); await archived("issue_thread_interaction", newest.id); await archived("issue_thread_interaction", independent.id); await quiet();
|
||||
await change(tx => tx.update(issueThreadInteractions).set({ status: "cancelled" }).where(eq(issueThreadInteractions.id, newest.id)));
|
||||
await archived("issue_thread_interaction", older.id); await quiet();
|
||||
});
|
||||
|
||||
it("excludes maintained reviews until a pending approval creates human attention", async () => {
|
||||
const row = await insertIssue("in_review", { assigneeAgentId: agentId });
|
||||
const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: row.id,
|
||||
kind: "request_confirmation", addresseeAgentId: agentId, payload: { prompt: "Agent verdict" } }).returning();
|
||||
expect((await loadAll()).some(entry => entry.sourceKind === "review")).toBe(false);
|
||||
const [approval] = await db.insert(approvals).values({ companyId, type: "hire_agent", payload: {}, status: "pending" }).returning();
|
||||
await db.insert(issueApprovals).values({ companyId, issueId: row.id, approvalId: approval.id });
|
||||
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
|
||||
expect(feed.items.some(item => item.sourceKind === "review" && item.subject.id === row.id)).toBe(true);
|
||||
await start(); await archived("review", row.id); await quiet();
|
||||
expect(await state("issue_thread_interaction", interaction.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("reactivates an exhausted failed run through the ordinary durable event producer", async () => {
|
||||
const origin = await insertIssue();
|
||||
const [run, newer] = await db.insert(heartbeatRuns).values([
|
||||
{ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() },
|
||||
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: origin.id }, createdAt: new Date(old().getTime() + 1000), finishedAt: old() },
|
||||
]).returning();
|
||||
await start(); await quiet();
|
||||
await appendHeartbeatRunEvent(db, { companyId, agentId, runId: run.id, eventType: "lifecycle", message: "Bounded retry exhausted after 3 attempts" });
|
||||
await quiet(); expect(await state("failed_run", run.id)).toBeNull();
|
||||
await change(tx => tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, newer.id)));
|
||||
await archived("failed_run", run.id); await quiet();
|
||||
});
|
||||
|
||||
it("cancels an agent error deadline through ordinary clearError", async () => {
|
||||
await db.update(agents).set({ status: "error", updatedAt: new Date() }).where(eq(agents.id, agentId));
|
||||
await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
await createAgentLifecycle(db).clearError(agentId); await quiet();
|
||||
expect(await state("agent_error_alert", agentId)).toBeNull();
|
||||
});
|
||||
|
||||
it("drains a paged backlog with source-scoped self notifications and leaves completed history quiet", async () => {
|
||||
await db.insert(issues).values(Array.from({ length: 225 }, () => ({ companyId, title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() })));
|
||||
const service = attentionRetentionService(db);
|
||||
const load = vi.spyOn(service, "load");
|
||||
await start(service);
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(225); }, { timeout: 30_000 });
|
||||
await quiet();
|
||||
const scopes = load.mock.calls.map(([scope]) => scope);
|
||||
expect(scopes.filter(scope => !scope)).toHaveLength(3);
|
||||
expect(scopes.filter(scope => scope && !scope.sourceId)).toHaveLength(0);
|
||||
expect(scopes.filter(scope => scope?.sourceId).length).toBeLessThanOrEqual(225);
|
||||
expect(new Set((await db.select().from(decisionRetention)).map(row => row.sourceId)).size).toBe(225);
|
||||
});
|
||||
|
||||
it("retries a failed atomic archive across drain and preserves notifications during a scan", async () => {
|
||||
const row = await insertIssue("in_review", { assigneeUserId: "board" });
|
||||
await db.execute(sql`create function attention_retention_fail() returns trigger language plpgsql as $$ begin raise exception 'retention injected'; end $$`);
|
||||
await db.execute(sql`create trigger attention_retention_fail before insert on decision_retention for each row execute function attention_retention_fail()`);
|
||||
await start(); expect(errors).toHaveLength(1); errors.length = 0;
|
||||
expect(await state("review", row.id)).toBeNull(); expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
draining = true; await db.execute(sql`drop trigger attention_retention_fail on decision_retention`);
|
||||
statements.length = 0; await vi.advanceTimersByTimeAsync(120_000);
|
||||
expect(statements.length).toBeLessThanOrEqual(1);
|
||||
expect(statements.every(query => query.includes("pg_xact_status"))).toBe(true);
|
||||
statements.length = 0; await vi.advanceTimersByTimeAsync(300_000); expect(statements).toEqual([]);
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", row.id); await quiet();
|
||||
await coordinator!.stop(); coordinator = undefined;
|
||||
let release!: () => void; let scanned!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
const entered = new Promise<void>(resolve => { scanned = resolve; });
|
||||
const service = attentionRetentionService(db), original = service.load;
|
||||
vi.spyOn(service, "load").mockImplementationOnce(async (...args) => { const result = await original(...args); scanned(); await gate; return result; });
|
||||
const starting = start(service); await entered;
|
||||
const next = await change(async tx => insertIssue("in_review", { assigneeUserId: "board" }, tx));
|
||||
release(); await starting; await archived("review", next.id); await quiet();
|
||||
});
|
||||
|
||||
it("registers and cancels a deadline through ordinary interaction creation and withdrawal", async () => {
|
||||
const origin = await insertIssue(); await start(); await quiet();
|
||||
const service = issueThreadInteractionService(db);
|
||||
const interaction = await service.create(origin, { kind: "request_confirmation", payload: { version: 1, prompt: "Approve the result?" } }, { userId: "board" });
|
||||
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(await state("issue_thread_interaction", interaction.id)).toBeNull();
|
||||
await service.withdrawInteraction(origin, interaction.id, { reason: "No longer needed" }, { userId: "board" }); await quiet();
|
||||
});
|
||||
|
||||
});
|
||||
@@ -62,7 +62,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-attention-service-");
|
||||
db = createDb(externalUrl ?? tempDb!.connectionString);
|
||||
db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 30_000);
|
||||
|
||||
afterEach(async () => {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, projects, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
vi.mock("../services/activity-log.js", () => ({ logActivity: vi.fn() }));
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("budget history for deleted scopes", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-budget-history-");
|
||||
db = createDb(database.connectionString);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterEach(async () => {
|
||||
await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents);
|
||||
await db.delete(projects); await db.delete(agents); await db.delete(companies);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "History company", issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Current agent", role: "engineer" });
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 300 }).returning();
|
||||
await db.insert(costEvents).values({ companyId, agentId, provider: "openai", model: "history-fixture", inputTokens: 1, outputTokens: 1, costCents: 17, occurredAt: new Date() });
|
||||
return { companyId, agentId, policy: policy! };
|
||||
}
|
||||
|
||||
it.each(["agent", "project"] as const)("keeps current budgets and deleted %s policy/incident history readable without changing stored rows", async scopeType => {
|
||||
const { companyId, policy } = await fixture(), deletedId = randomUUID();
|
||||
if (scopeType === "agent") await db.insert(agents).values({ id: deletedId, companyId, name: "Former agent", role: "engineer" });
|
||||
else await db.insert(projects).values({ id: deletedId, companyId, name: "Former project" });
|
||||
const [historical] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: deletedId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 200 }).returning();
|
||||
await db.insert(budgetIncidents).values({ companyId, policyId: historical!.id, scopeType, scopeId: deletedId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 200, amountObserved: 201 });
|
||||
if (scopeType === "agent") await db.delete(agents).where(eq(agents.id, deletedId));
|
||||
else await db.delete(projects).where(eq(projects.id, deletedId));
|
||||
const policiesBefore = await db.select().from(budgetPolicies), incidentsBefore = await db.select().from(budgetIncidents), costsBefore = await db.select().from(costEvents);
|
||||
|
||||
const overview = await budgetService(db).overview(companyId);
|
||||
expect(overview.policies).toHaveLength(2);
|
||||
expect(overview.policies.find(row => row.policyId === policy.id)).toMatchObject({ companyId, scopeName: "Current agent", amount: 300, observedAmount: 17 });
|
||||
expect(overview.policies.find(row => row.policyId === historical!.id)).toMatchObject({ companyId, scopeType, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, amount: 200 });
|
||||
expect(overview.activeIncidents).toMatchObject([{ companyId, policyId: historical!.id, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, status: "open", amountObserved: 201 }]);
|
||||
expect(await db.select().from(budgetPolicies)).toEqual(policiesBefore);
|
||||
expect(await db.select().from(budgetIncidents)).toEqual(incidentsBefore);
|
||||
expect(await db.select().from(costEvents)).toEqual(costsBefore);
|
||||
});
|
||||
it("rejects missing-scope budget grants but can dismiss their historical incidents", async () => {
|
||||
const { companyId, agentId } = await fixture(), missingId = randomUUID(), service = budgetService(db);
|
||||
await expect(service.evaluateAdmission(companyId, missingId)).rejects.toThrow("Agent not found");
|
||||
await expect(service.evaluateAdmission(companyId, agentId, { projectId: missingId })).rejects.toThrow("Project not found");
|
||||
for (const scopeType of ["agent", "project"] as const) {
|
||||
await expect(service.upsertPolicy(companyId, { scopeType, scopeId: missingId, amount: 100 }, "board")).rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: missingId,
|
||||
windowKind: "calendar_month_utc", amount: 100 }).returning();
|
||||
const [approval] = await db.insert(approvals).values({ companyId, type: "budget_override_required", payload: {} }).returning();
|
||||
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, approvalId: approval!.id, scopeType, scopeId: missingId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 100, amountObserved: 101 }).returning();
|
||||
const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) };
|
||||
await expect(service.resolveIncident(companyId, incident!.id, { action: "raise_budget_and_resume", amount: 200 }, "board"))
|
||||
.rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
|
||||
expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }).toEqual(before);
|
||||
const dismissed = await service.resolveIncident(companyId, incident!.id, { action: "keep_paused" }, "board");
|
||||
expect(dismissed).toMatchObject({ companyId, policyId: policy!.id, scopeId: missingId, scopeName: `Deleted ${scopeType}`, status: "dismissed" });
|
||||
const { status: _status, resolvedAt: _resolvedAt, updatedAt: _updatedAt, ...preserved } = incident!;
|
||||
const [saved] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident!.id));
|
||||
expect(saved).toMatchObject({ ...preserved, status: "dismissed", resolvedAt: expect.any(Date) });
|
||||
expect((await db.select().from(approvals).where(eq(approvals.id, approval!.id)))[0]).toMatchObject({ status: "rejected" });
|
||||
expect(await db.select().from(budgetPolicies)).toEqual(before.policies);
|
||||
expect(await db.select().from(agents)).toEqual(before.agents);
|
||||
expect(await db.select().from(costEvents)).toEqual(before.costs);
|
||||
expect((await service.overview(companyId)).activeIncidents.some(row => row.id === incident!.id)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a corrupted policy pointing to another company's scope instead of exposing its name", async () => {
|
||||
const { companyId } = await fixture(), foreignCompanyId = randomUUID(), foreignAgentId = randomUUID();
|
||||
await db.insert(companies).values({ id: foreignCompanyId, name: "Other company", issuePrefix: `F${foreignCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: foreignAgentId, companyId: foreignCompanyId, name: "Private other-company agent", role: "engineer" });
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: foreignAgentId, windowKind: "calendar_month_utc", amount: 200 }).returning();
|
||||
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, scopeType: "agent", scopeId: foreignAgentId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 200, amountObserved: 201 }).returning();
|
||||
const service = budgetService(db);
|
||||
await expect(service.overview(companyId)).rejects.toThrow("Budget scope not found");
|
||||
for (const scopeId of [foreignAgentId, randomUUID()]) {
|
||||
await db.update(budgetIncidents).set({ scopeId }).where(eq(budgetIncidents.id, incident!.id));
|
||||
const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents) };
|
||||
for (const action of ["raise_budget_and_resume", "keep_paused"] as const) {
|
||||
await expect(service.resolveIncident(companyId, incident!.id, { action, amount: 300 }, "board")).rejects.toThrow("Budget scope not found");
|
||||
expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents) }).toEqual(before);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
@@ -16341,9 +16341,15 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
await tx.update(toolConnections).set({ credentialSecretRefs: connection.credentialSecretRefs.map(ref => ({ ...ref,
|
||||
versionSelector: ref.versionSelector === "1" ? "latest" : "1" })) }).where(eq(toolConnections.id, connection.id));
|
||||
});
|
||||
await expect.poll(async () => ((await f.checkpoint()).value as Record<string, unknown>).credentialFingerprint,
|
||||
{ timeout: 5000 }).not.toBe(oldWindow.credentialFingerprint);
|
||||
expect((await f.checkpoint()).value).toMatchObject({ generation: oldWindow.generation, outcome: "epoch_initialized" });
|
||||
// Credential synchronization and epoch initialization commit separately.
|
||||
// Require the final epoch receipt, not just the intermediate fingerprint.
|
||||
await expect.poll(async () => {
|
||||
const value = (await f.checkpoint()).value as Record<string, unknown>;
|
||||
return { ...value, fingerprintChanged: typeof value.credentialFingerprint === "string"
|
||||
&& value.credentialFingerprint !== oldWindow.credentialFingerprint };
|
||||
}, { timeout: 5000 }).toMatchObject({
|
||||
fingerprintChanged: true, generation: oldWindow.generation, outcome: "epoch_initialized",
|
||||
});
|
||||
expect(f.requests).toEqual([]);
|
||||
} finally { await coordinator.stop(); }
|
||||
});
|
||||
|
||||
@@ -4168,6 +4168,7 @@ describe("company portability", () => {
|
||||
const insertedRelationValues: Array<Record<string, unknown>> = [];
|
||||
const monitorUpdates: Array<Record<string, unknown>> = [];
|
||||
const db = {
|
||||
transaction: async (write: (tx: unknown) => Promise<unknown>) => write(db),
|
||||
insert: () => ({
|
||||
values: (rows: Array<Record<string, unknown>>) => ({
|
||||
onConflictDoNothing: async () => {
|
||||
@@ -4184,6 +4185,7 @@ describe("company portability", () => {
|
||||
}),
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any;
|
||||
db.transaction = async (write: (tx: typeof db) => Promise<unknown>) => write(db);
|
||||
return { db, insertedRelationValues, monitorUpdates };
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
@@ -28,12 +29,13 @@ import {
|
||||
import { issueReadSqlCondition, projectReadSqlCondition } from "../services/authorization.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_COMPANY_SEARCH_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres company search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
`Skipping embedded Postgres company search tests on this host: ${("reason" in embeddedPostgresSupport ? embeddedPostgresSupport.reason : "unsupported environment")}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -80,8 +82,11 @@ describeEmbeddedPostgres("companySearchService", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-");
|
||||
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
// External suites can reuse this dedicated database sequentially. Match
|
||||
// the fresh embedded fixture before this suite creates its own rows.
|
||||
if (externalDatabaseUrl) await db.execute(sql`TRUNCATE TABLE companies CASCADE`);
|
||||
svc = companySearchService(db);
|
||||
await db.execute(sql.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm"));
|
||||
}, 20_000);
|
||||
@@ -100,6 +105,7 @@ describeEmbeddedPostgres("companySearchService", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { installDatabaseWorkSignals, signalDatabaseWork, databaseWorkPending } from "../../../packages/db/src/work-signals.js";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { registerDeadlineWork } from "../services/deadline-work.js";
|
||||
@@ -39,3 +40,132 @@ it.each([
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("releases a settled first-write hold during drain and retains a probe-free resume wake", async () => {
|
||||
vi.useFakeTimers();
|
||||
let draining = false;
|
||||
const owner = installDatabaseWorkSignals({
|
||||
execute: vi.fn(async () => [{ status: "committed" }]),
|
||||
transaction: async (callback: (tx: object) => Promise<unknown>) => callback({ execute: async () => [{ xid: "1" }] }),
|
||||
});
|
||||
const run = vi.fn(async () => {}), next = vi.fn(async () => null);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() });
|
||||
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 });
|
||||
await worker.ready;
|
||||
draining = true;
|
||||
const before = idleWorkSnapshot().active;
|
||||
await owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(idleWorkSnapshot().active).toBe(before);
|
||||
expect(coordinator.nextWakeAt()).not.toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(3 * 30_000);
|
||||
expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled();
|
||||
draining = false;
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
|
||||
it("retains failed work through drain and preserves a future domain deadline", async () => {
|
||||
vi.useFakeTimers();
|
||||
let draining = false;
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
|
||||
const run = vi.fn(async () => { if (run.mock.calls.length === 1) { draining = true; throw new Error("retry debt"); } });
|
||||
const deadline = Date.now() + 600_000;
|
||||
const next = vi.fn(async () => deadline);
|
||||
const onError = vi.fn();
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError });
|
||||
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 });
|
||||
await worker.ready;
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(run).toHaveBeenCalledOnce(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled();
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBe(deadline);
|
||||
draining = true; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(coordinator.nextWakeAt()).toBe(deadline); expect(onError).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("does no domain probes in warm standby and reconstructs after admission opens", async () => {
|
||||
vi.useFakeTimers();
|
||||
let standby = true;
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
|
||||
const run = vi.fn(async () => {}), next = vi.fn(async () => null);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !standby, canReconcile: () => !standby, onError: vi.fn() });
|
||||
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !standby, retryMs: 30_000 });
|
||||
await worker.ready; await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(run).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled();
|
||||
standby = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(run).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
|
||||
it("does not release an ambiguous database intent while dispatch is drained", async () => {
|
||||
vi.useFakeTimers();
|
||||
let draining = false, committed = false;
|
||||
const owner = installDatabaseWorkSignals({
|
||||
execute: vi.fn(async () => [{ status: committed ? "committed" : "in progress" }]),
|
||||
transaction: async (callback: (tx: object) => Promise<unknown>) => {
|
||||
await callback({ execute: async () => [{ xid: "2" }] });
|
||||
throw new Error("lost acknowledgement");
|
||||
},
|
||||
});
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() });
|
||||
const run = vi.fn(async () => {});
|
||||
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next: async () => null, canRun: () => !draining, retryMs: 30_000 });
|
||||
await worker.ready; draining = true;
|
||||
const before = idleWorkSnapshot().active;
|
||||
await expect(owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); })).rejects.toThrow("lost acknowledgement");
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(idleWorkSnapshot().active).toBe(before + 1);
|
||||
expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(true);
|
||||
committed = true; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(false);
|
||||
expect(idleWorkSnapshot().active).toBe(before);
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
|
||||
it("backs off an expired deadline when only the domain is drained and resumes promptly", async () => {
|
||||
vi.useFakeTimers();
|
||||
let paused = false;
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
|
||||
const run = vi.fn(async () => {});
|
||||
const domainCanRun = vi.fn(() => !paused);
|
||||
const deadline = Date.now() + 1_000;
|
||||
const next = vi.fn(async () => run.mock.calls.length === 1 ? deadline : null);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() });
|
||||
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, {
|
||||
run, next, canRun: domainCanRun, retryMs: 30_000,
|
||||
});
|
||||
await worker.ready;
|
||||
paused = true;
|
||||
const before = idleWorkSnapshot();
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000);
|
||||
const attempts = domainCanRun.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(domainCanRun.mock.calls.length - attempts).toBeLessThanOrEqual(12);
|
||||
// Four attempts (expiry plus three retries), each begins and ends tracked work.
|
||||
expect(idleWorkSnapshot().generation - before.generation).toBe(8);
|
||||
expect(idleWorkSnapshot().active).toBe(before.active);
|
||||
expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled();
|
||||
paused = false;
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
|
||||
it("retains a prompt paused wake when a new request preempts a far-future deadline", async () => {
|
||||
vi.useFakeTimers();
|
||||
let paused = false;
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
|
||||
const run = vi.fn(async () => {});
|
||||
const deadline = Date.now() + 86_400_000;
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() });
|
||||
const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, {
|
||||
run, next: async () => run.mock.calls.length === 1 ? deadline : null, canRun: () => !paused, retryMs: 30_000,
|
||||
});
|
||||
await worker.ready; paused = true; worker.wake();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000);
|
||||
expect(owner.execute).not.toHaveBeenCalled();
|
||||
paused = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
@@ -73,7 +73,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes
|
||||
const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({
|
||||
status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId,
|
||||
updatedAt, inputValues: {}, metadata: { continuationPending: true },
|
||||
}).where(eq(decisions.id, id)));
|
||||
}).where(eq(decisions.id, id)), { companyId });
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
@@ -467,7 +467,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes
|
||||
} else {
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running",
|
||||
chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000),
|
||||
metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)));
|
||||
metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)), { companyId });
|
||||
}
|
||||
await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet();
|
||||
const [retained] = await db.select().from(decisionRetention);
|
||||
|
||||
@@ -793,7 +793,7 @@ describePg("decisionService", () => {
|
||||
expect((await service().get(id))?.metadata.continuationPending).toBe(false);
|
||||
}
|
||||
const markReady = (id: string, source = db) => writeDecisionChange(source, async tx => tx.update(decisions)
|
||||
.set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)));
|
||||
.set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)), { companyId });
|
||||
|
||||
it("coalesces sequential tenant writes without dropping pending recovery", async () => {
|
||||
const row = await createCommentDecision();
|
||||
|
||||
@@ -0,0 +1,427 @@
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
|
||||
import { accessService } from "../services/access.js";
|
||||
import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js";
|
||||
import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js";
|
||||
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
|
||||
import { issueTreeControlService } from "../services/issue-tree-control.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyNativeStatusWakeWork, notifyTaskRetrySettlement } from "../services/deferred-execution-notifications.js";
|
||||
import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-receipt-work.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { requestTaskRetry } from "../services/task-retry.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
describe("deferred receipt durable dispatch", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
let drain = false;
|
||||
const errors: unknown[] = [];
|
||||
const passes = new Set<Promise<unknown>>();
|
||||
beforeAll(async () => {
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL");
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0;
|
||||
heartbeat = heartbeatService(db);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const pass = Promise.resolve(task.run(signal)); passes.add(pass);
|
||||
void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass;
|
||||
} });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
|
||||
vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), actorId = "deferred-receipt-user";
|
||||
await db.insert(authUsers).values({ id: actorId, name: "Receipt operator", email: `${actorId}@example.test`,
|
||||
createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companies).values({ id: companyId, name: "Deferred dispatch", issuePrefix: `DE${companyId.slice(0, 6)}`,
|
||||
defaultResponsibleUserId: actorId, requireBoardApprovalForNewAgents: false });
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: actorId, principalType: "user", status: "active", membershipRole: "owner" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Receipt process", adapterType: "process", status: "idle",
|
||||
adapterConfig: { command: process.execPath, args: ["-e", "console.log('receipt-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Resume saved task", status: "todo", assigneeAgentId: agent.id }).returning();
|
||||
const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, issueId: issue.id,
|
||||
status: "failed", finishedAt: new Date(), responsibleUserId: actorId, errorCode: "process_failed",
|
||||
contextSnapshot: { issueId: issue.id }, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
|
||||
return { companyId, agent, issue, source, actorId };
|
||||
}
|
||||
async function quiet() {
|
||||
await vi.waitFor(async () => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
|
||||
const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute");
|
||||
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed);
|
||||
}
|
||||
async function finish(issueId: string) {
|
||||
await issueService(db).update(issueId, { status: "done" });
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
|
||||
}, { timeout: 15_000 });
|
||||
const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"));
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
if (saved.length) {
|
||||
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
|
||||
while (Date.now() <= due) {
|
||||
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
|
||||
await Promise.allSettled([...passes]); await immediate();
|
||||
}
|
||||
}
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(500); await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0);
|
||||
});
|
||||
}
|
||||
it("is quiet after empty reconstruction and terminal receipt history", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet();
|
||||
});
|
||||
it("dispatches an ordinary committed task Retry into saved process output", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
expect(receipt.accepted).toBe(true);
|
||||
let runId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId!;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
await finish(f.issue.id);
|
||||
const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`));
|
||||
expect(linked).toHaveLength(2);
|
||||
expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId]));
|
||||
expect(linked.map(row => row.idempotencyKey)).toContain(`task-retry-dispatch:${receipt.requestId}`);
|
||||
expect(linked.find(row => row.id === receipt.requestId)?.status).toBe("coalesced");
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1);
|
||||
await quiet();
|
||||
});
|
||||
it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => {
|
||||
const f = await fixture(), updatedAt = new Date(), native = kind === "native claim";
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "automation", status: native ? "claimed" : "deferred_issue_execution", claimedAt: updatedAt, updatedAt,
|
||||
payload: native ? {} : { executionWait: { reason: "recovery" } },
|
||||
requestedByActorType: "system", requestedByActorId: native ? "native-status-committer" : null }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const check = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
|
||||
await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, scope.requestId!));
|
||||
});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: check, settleTaskChat: vi.fn(), dispatchNative: check,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const due = updatedAt.getTime() + (native ? 60_000 : 30_000);
|
||||
expect(coordinator.nextWakeAt()).toBe(due); expect(check).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(check).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await vi.waitFor(() => expect(check).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id })));
|
||||
await quiet();
|
||||
});
|
||||
it("reconciles the first drained producer and waits for resume without domain SQL", async () => {
|
||||
const f = await fixture(); drain = true;
|
||||
const resume = vi.fn(async () => {}), dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await db.transaction(async tx => {
|
||||
await notifyDeferredExecutionWork(tx, { companyId: f.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, source: "on_demand",
|
||||
status: "deferred_issue_execution", reason: "task_retry" });
|
||||
});
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(resume).not.toHaveBeenCalled(); expect(dispatchNative).not.toHaveBeenCalled();
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
drain = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(1)); await quiet();
|
||||
});
|
||||
it.each([
|
||||
{ name: "saved comment", payload: { _paperclipWakeContext: { wakeCommentIds: ["saved-comment"] } } },
|
||||
{ name: "interaction", payload: { mutation: "interaction" } },
|
||||
{ name: "execution wait", payload: { executionWait: { reason: "recovery" } } },
|
||||
{ name: "interrupt", payload: { queuedCommentInterrupt: { actorId: "operator" } } },
|
||||
])("activates only the committed company's $name receipt after an empty start", async ({ payload }) => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
const resume = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const [other] = await db.insert(agentWakeupRequests).values({ companyId: foreign.companyId, agentId: foreign.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload }).returning();
|
||||
let requestId: string;
|
||||
await db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await notifyDeferredExecutionWork(nested, { companyId: f.companyId });
|
||||
const [row] = await nested.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: new Date(Date.now() - 31_000) }).returning();
|
||||
requestId = row.id;
|
||||
});
|
||||
expect(resume).not.toHaveBeenCalled();
|
||||
});
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId })), { timeout: 5_000 });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, other.id)))[0].status).toBe("deferred_issue_execution");
|
||||
await quiet();
|
||||
});
|
||||
it("reconstructs a cancelled legacy run's exact deferred queue even without comment payload", async () => {
|
||||
const f = await fixture();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution" }).returning();
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { queuedCommentInterruptQueueId: wake.id } })
|
||||
.where(eq(heartbeatRuns.id, f.source.id));
|
||||
const resume = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id }));
|
||||
await quiet();
|
||||
});
|
||||
it("settles a processed chat receipt without restaging it until an actual eligibility commit", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Receipt chat", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Receipt chat", transport: "rest_api" }).returning();
|
||||
const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection.id,
|
||||
provider: "slack", publicId: randomUUID(), assignedAgentId: f.agent.id }).returning();
|
||||
const intentId = randomUUID(), actionId = randomUUID(), previous = new Date(Date.now() - 60_000);
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
{ id: intentId, companyId: f.companyId, agentId: f.agent.id, source: "on_demand", reason: "task_retry",
|
||||
status: "deferred_issue_execution", updatedAt: previous, payload: { issueId: f.issue.id,
|
||||
taskRetry: { schema: "paperclip.task-retry.v1", generationId: intentId, sourceRunId: f.source.id,
|
||||
actorId: f.actorId, strategy: "evaluate", chatActionId: actionId } } },
|
||||
{ id: actionId, companyId: f.companyId, agentId: f.agent.id, source: "automation", status: "deferred_issue_execution",
|
||||
payload: { issueId: f.issue.id, taskRetryRequestId: intentId } },
|
||||
]);
|
||||
await db.transaction(async tx => {
|
||||
await notifyTaskRetrySettlement(tx, { companyId: f.companyId, requestId: intentId });
|
||||
await tx.insert(chatActions).values({ id: actionId, companyId: f.companyId, endpointId: endpoint.id,
|
||||
providerActionId: randomUUID(), kind: "failed_run_retry", status: "processed", updatedAt: new Date(),
|
||||
payload: { issueId: f.issue.id, taskRetryIntentId: intentId, failedRunId: f.source.id } });
|
||||
});
|
||||
await vi.waitFor(async () => {
|
||||
const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, intentId));
|
||||
expect(intent.updatedAt.getTime()).toBeGreaterThan(previous.getTime());
|
||||
}, { timeout: 5_000 });
|
||||
expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status).toBe("processed");
|
||||
await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
|
||||
await vi.waitFor(async () => expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status)
|
||||
.toBe("issued"), { timeout: 5_000 });
|
||||
await quiet();
|
||||
});
|
||||
it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => {
|
||||
const f = await fixture(), initialAt = new Date();
|
||||
const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({
|
||||
companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId,
|
||||
}))).returning();
|
||||
const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId },
|
||||
_paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } };
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: initialAt }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps);
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: scope => queue.resumeDeferredExecutionReceipts(scope),
|
||||
settleTaskChat: async () => {}, dispatchNative: async () => {}, schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
|
||||
const movedAt = new Date();
|
||||
const writer = createQueuedCommentIssueLockWriter(db, { syncCommentReferences: async () => {},
|
||||
deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {} });
|
||||
await writer.withLockedQueue({ issue: f.issue, queueId: wake.id,
|
||||
actor: { actorType: "user", actorId: f.actorId, agentId: null, runId: null, agentApiKeyId: null } },
|
||||
async (_locked, tx) => { await tx.updateWakeQueuedCommentIds({ wakeId: wake.id, payload,
|
||||
ids: comments.map(row => row.id).reverse(), updatedAt: movedAt }); });
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
expect(deps.resumeQueuedCommentInterrupt).not.toHaveBeenCalled();
|
||||
expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(deps.resumeQueuedCommentInterrupt).toHaveBeenCalledExactlyOnceWith(f.companyId, wake.id));
|
||||
await quiet();
|
||||
});
|
||||
it("reactivates a held task Retry when the authorized subtree hold is released without wakeAgents", async () => {
|
||||
const f = await fixture(), tree = issueTreeControlService(db);
|
||||
const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
|
||||
const hold = await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
expect(request.requestId).toBeTruthy();
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.payload?.executionWait).toMatchObject({ reason: "issue_tree_hold_active" });
|
||||
expect(wake.runId).toBeNull();
|
||||
}, { timeout: 5_000 });
|
||||
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
|
||||
await tree.releaseHold(f.companyId, f.issue.id, hold.hold.id, { actor, reason: "Resume saved Retry" });
|
||||
let runId: string | null = null;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
|
||||
}, { timeout: 40_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id); await quiet();
|
||||
});
|
||||
it("keeps a moved native dispatch error deadline instead of retrying at its old time", async () => {
|
||||
const f = await fixture(), initialAt = new Date();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "automation", status: "queued", error: "Transient dispatch failure", updatedAt: initialAt,
|
||||
requestedByActorType: "system", requestedByActorId: "native-status-committer" }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: async () => {}, settleTaskChat: async () => {},
|
||||
dispatchNative, schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
|
||||
const movedAt = new Date();
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: movedAt }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
expect(dispatchNative).not.toHaveBeenCalled(); expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(dispatchNative).toHaveBeenCalledTimes(1)); await quiet();
|
||||
});
|
||||
it("arms one new wait reason grace and stays quiet when that same gate is still closed", async () => {
|
||||
const f = await fixture(); let reason = "operator_pause";
|
||||
const resume = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
|
||||
await writeDeferredExecutionWait(db, { companyId: scope.companyId!, requestId: scope.requestId! }, { reason });
|
||||
});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {},
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", reason: "task_retry" }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
for (const nextReason of ["operator_pause", "budget_pause"]) {
|
||||
reason = nextReason;
|
||||
await writeDeferredExecutionWait(db, { companyId: f.companyId, requestId: wake.id }, { reason });
|
||||
const [current] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
const due = current.updatedAt.getTime() + 30_000;
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due));
|
||||
const checked = resume.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now());
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(checked + 1));
|
||||
await quiet();
|
||||
}
|
||||
expect(resume).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("reactivates a saved Retry through ordinary restore cancelled task", async () => {
|
||||
const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
const tree = issueTreeControlService(db);
|
||||
const cancel = await tree.createHold(f.companyId, f.issue.id, { mode: "cancel", actor });
|
||||
await tree.cancelIssueStatusesForHold(f.companyId, f.issue.id, cancel.hold.id);
|
||||
const restore = await tree.createHold(f.companyId, f.issue.id, { mode: "restore", actor });
|
||||
const commitEligibility = () => tree.restoreIssueStatusesForHold(f.companyId, f.issue.id, restore.hold.id, { actor });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull();
|
||||
await commitEligibility();
|
||||
let runId: string | null = null;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id); await quiet();
|
||||
});
|
||||
it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => {
|
||||
const f = await fixture(), departing = `departing-${randomUUID()}`;
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
await db.insert(authUsers).values({ id: departing, name: "Departing member", email: `${departing}@example.test`,
|
||||
createdAt: new Date(), updatedAt: new Date() });
|
||||
const [member] = await db.insert(companyMemberships).values({ companyId: f.companyId, principalId: departing,
|
||||
principalType: "user", status: "active", membershipRole: "member" }).returning();
|
||||
await issueService(db).update(f.issue.id, { assigneeUserId: departing, assigneeAgentId: null });
|
||||
const readIntent = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0];
|
||||
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
|
||||
const deferred = vi.fn(), native = vi.fn();
|
||||
const unsubscribes = [subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred),
|
||||
subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents, native)];
|
||||
try {
|
||||
await expect(db.transaction(async tx => {
|
||||
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
|
||||
{ reassignment: { assigneeAgentId: f.agent.id } });
|
||||
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
|
||||
throw new Error("roll back member archive");
|
||||
})).rejects.toThrow("roll back member archive");
|
||||
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].assigneeUserId).toBe(departing);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.deferredExecutionReceipts);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents);
|
||||
deferred.mockClear(); native.mockClear();
|
||||
await db.transaction(async tx => {
|
||||
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
|
||||
{ reassignment: { assigneeAgentId: f.agent.id } });
|
||||
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
|
||||
});
|
||||
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({
|
||||
assigneeUserId: null, assigneeAgentId: f.agent.id,
|
||||
});
|
||||
expect((await db.select().from(companyMemberships).where(eq(companyMemberships.id, member.id)))[0].status).toBe("archived");
|
||||
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0);
|
||||
} finally { for (const unsubscribe of unsubscribes) unsubscribe(); }
|
||||
await quiet();
|
||||
});
|
||||
it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => {
|
||||
const f = await fixture(); const dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await expect(db.transaction(async tx => {
|
||||
await notifyNativeStatusWakeWork(tx, { companyId: f.companyId });
|
||||
throw new Error("outer rollback");
|
||||
})).rejects.toThrow("outer rollback");
|
||||
await db.transaction(async tx => {
|
||||
await expect(tx.transaction(async nested => {
|
||||
await notifyNativeStatusWakeWork(nested, { companyId: f.companyId }); throw new Error("nested rollback");
|
||||
})).rejects.toThrow("nested rollback");
|
||||
});
|
||||
await quiet(); expect(dispatchNative).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -73,7 +73,7 @@ describeEmbeddedPostgres("git restore point retention", () => {
|
||||
} = {}) {
|
||||
const companyId = input.companyId ?? randomUUID();
|
||||
if (!input.companyId) {
|
||||
await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.slice(0, 6)}` });
|
||||
await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.replaceAll("-", "")}` });
|
||||
}
|
||||
const issueStatus = input.issueStatus ?? "done";
|
||||
const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS);
|
||||
|
||||
@@ -19,11 +19,20 @@ import { secretService } from "../services/secrets.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { aiConnectionService } from "../services/ai-connections.js";
|
||||
import * as aiRuntime from "../services/ai-connection-runtime.js";
|
||||
import * as connectorRuntime from "../services/connector-runtime.js";
|
||||
import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js";
|
||||
import * as sentry from "../sentry.js";
|
||||
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
|
||||
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>(complete => { resolve = complete; });
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
const skillFailures = vi.hoisted(() => new Map<string, string>());
|
||||
const skillWaits = vi.hoisted(() => new Map<string, () => Promise<void>>());
|
||||
|
||||
describe("Runner home mode during recovery preparation", () => {
|
||||
it.each([
|
||||
@@ -52,6 +61,7 @@ vi.mock("../services/company-skills.js", async importOriginal => {
|
||||
return { ...service, withRuntimeSkillEntries: async <T>(companyId: string,
|
||||
options: Parameters<typeof service.withRuntimeSkillEntries>[1],
|
||||
consume: (entries: Awaited<ReturnType<typeof service.listRuntimeSkillEntries>>) => T | Promise<T>) => {
|
||||
await skillWaits.get(companyId)?.();
|
||||
const failure = skillFailures.get(companyId);
|
||||
if (failure) throw new Error(failure);
|
||||
return service.withRuntimeSkillEntries(companyId, options, consume);
|
||||
@@ -91,7 +101,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }),
|
||||
});
|
||||
}, 60_000);
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); });
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); skillWaits.clear(); });
|
||||
afterAll(async () => {
|
||||
await heartbeat?.drainActiveRunExecutions();
|
||||
unregisterServerAdapter(adapterType);
|
||||
@@ -288,6 +298,45 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } });
|
||||
}, 60_000);
|
||||
it("overlaps skills with assignments and dependent connection instructions before composition", async () => {
|
||||
const f = await seed();
|
||||
const skillGate = deferred<void>();
|
||||
skillWaits.set(f.companyId, () => skillGate.promise);
|
||||
const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments");
|
||||
const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun");
|
||||
const configured = await configuration(f, { provider: "codex", model: "gpt-6.1-sol", homeMode: "isolated" });
|
||||
const opening = configured.prepare();
|
||||
try {
|
||||
await vi.waitFor(() => {
|
||||
expect(assignments).toHaveBeenCalledOnce();
|
||||
expect(instructions).toHaveBeenCalledOnce();
|
||||
});
|
||||
} finally {
|
||||
skillGate.resolve();
|
||||
}
|
||||
expect(await opening).toMatchObject({ prepared: true, runtimeConfig: { homeMode: "isolated", model: "gpt-6.1-sol" } });
|
||||
});
|
||||
|
||||
it("settles concurrent configuration reads before exposing the first preparation failure", async () => {
|
||||
const f = await seed();
|
||||
const assignmentGate = deferred<Awaited<ReturnType<typeof connectorRuntime.resolveConnectorAssignments>>>();
|
||||
const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments").mockReturnValue(assignmentGate.promise);
|
||||
const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun").mockRejectedValue(new Error("later instruction failure"));
|
||||
skillFailures.set(f.companyId, "first skill failure");
|
||||
const configured = await configuration(f, { provider: "codex", homeMode: "isolated" });
|
||||
let settled = false;
|
||||
const opening = configured.prepare();
|
||||
void opening.then(() => { settled = true; }, () => { settled = true; });
|
||||
try {
|
||||
await vi.waitFor(() => expect(assignments).toHaveBeenCalledOnce());
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
expect(instructions).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
assignmentGate.resolve([]);
|
||||
}
|
||||
await expect(opening).rejects.toThrow("first skill failure");
|
||||
});
|
||||
|
||||
it("does not install new environment declarations when recovering a saved skill selection", async () => {
|
||||
const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`;
|
||||
@@ -445,11 +494,11 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
const runIdHash = createHash("sha256").update(finished.id).digest("hex").slice(0, 12);
|
||||
const steps = exporter.getFinishedSpans().filter(span => span.name.startsWith("heartbeat.prepare.configuration_")
|
||||
&& span.attributes["paperclip.task.run.run_id"] === runIdHash);
|
||||
expect(steps.map(span => span.name)).toEqual([
|
||||
expect(steps.map(span => span.name).sort()).toEqual([
|
||||
"configuration_mentions", "configuration_github_identity", "configuration_secrets", "configuration_ai_runtime",
|
||||
"configuration_skills", "configuration_connector_assignments", "configuration_connector_skills",
|
||||
"configuration_connector_delivery", "configuration_connection_instructions", "configuration_agent_identity",
|
||||
].map(step => `heartbeat.prepare.${step}`));
|
||||
].map(step => `heartbeat.prepare.${step}`).sort());
|
||||
for (const span of steps) {
|
||||
expect(span.status).toEqual({ code: 0 });
|
||||
expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
|
||||
@@ -494,7 +543,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
&& span.attributes["paperclip.task.run.run_id"] === runIdHash);
|
||||
expect(steps.find(span => span.name === "heartbeat.prepare.configuration_ai_runtime")?.status.code).toBe(0);
|
||||
expect(steps.find(span => span.name === "heartbeat.prepare.configuration_skills")?.status).toEqual({ code: 2 });
|
||||
expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(false);
|
||||
expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(true);
|
||||
for (const span of steps) {
|
||||
expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
|
||||
expect(span.events).toEqual([]);
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
|
||||
import { registerRunDispatchWork } from "../services/heartbeat/dispatch-work.js";
|
||||
|
||||
const cleanup: Array<() => Promise<void>> = [];
|
||||
afterEach(async () => {
|
||||
for (const stop of cleanup.splice(0)) await stop();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean } = {}) {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(1_000_000);
|
||||
const dialect = new PgDialect();
|
||||
const execute = vi.fn(async (query: any) => {
|
||||
if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
|
||||
throw new Error("unexpected database query");
|
||||
});
|
||||
const db: { execute: typeof execute; transaction: (callback: (tx: any) => Promise<any>) => Promise<any> } = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise<any>): Promise<any> {
|
||||
return callback({ execute, transaction: db.transaction });
|
||||
} });
|
||||
const state = { due: input.due ?? null as number | null, suppressed: input.suppressed ?? false,
|
||||
draining: input.draining ?? false, standby: false, fail: false };
|
||||
const promote = vi.fn(async () => {
|
||||
if (state.fail) throw new Error("database unavailable");
|
||||
if (state.due !== null && state.due <= Date.now()) state.due = null;
|
||||
});
|
||||
const next = vi.fn(async () => state.due);
|
||||
const dispatch = vi.fn(async (_scope?: { companyId: string }) => {});
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db,
|
||||
canRun: () => !state.draining && !state.standby, canReconcile: () => !state.standby, onError: vi.fn() });
|
||||
const worker = registerRunDispatchWork(coordinator, {
|
||||
promoteDueScheduledRetries: promote, nextScheduledRetryAt: next,
|
||||
resumeQueuedRunsAfterDrain: dispatch, schedulingSuppressed: async () => state.suppressed,
|
||||
}, () => !state.draining && !state.standby);
|
||||
cleanup.push(() => coordinator.stop());
|
||||
return { db, state, promote, next, dispatch, coordinator, worker,
|
||||
notify: (scope?: { companyId: string }) => db.transaction(tx => notifyRunDispatch(tx, scope)),
|
||||
settle: () => vi.advanceTimersByTimeAsync(1_000) };
|
||||
}
|
||||
|
||||
describe("run dispatch obligations", () => {
|
||||
it.each([false, true])("settles empty/feature-suppressed state and stops all owner work (suppressed=%s)", async suppressed => {
|
||||
const f = fixture({ suppressed }); await f.worker.ready;
|
||||
const calls = [f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length];
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect([f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]).toEqual(calls);
|
||||
expect(f.db.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
it("restores an exact deadline and does no earlier recovery work", async () => {
|
||||
const f = fixture({ due: 1_010_000 }); await f.worker.ready;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_010_000);
|
||||
await vi.advanceTimersByTimeAsync(9_999); expect(f.promote).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
expect(f.state.due).toBeNull(); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
expect(f.dispatch).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("rearms earlier work and cancels a future deadline on a committed producer change", async () => {
|
||||
const f = fixture({ due: 1_600_000 }); await f.worker.ready;
|
||||
f.state.due = 1_005_000; await f.notify(); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_005_000);
|
||||
f.state.due = null; await f.notify(); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const before = f.next.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.next).toHaveBeenCalledTimes(before);
|
||||
});
|
||||
it("batches repeated tenant writes and passes only their company to queued dispatch", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
for (let index = 0; index < 100; index++) {
|
||||
const source = { companyId: "company-a", issueId: `issue-${index % 2}`,
|
||||
runId: `run-${index}`, agentId: `agent-${index}`, interactionBody: "not a dispatch key" };
|
||||
await f.notify(source);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
}
|
||||
expect(f.promote.mock.calls.length).toBeLessThanOrEqual(2);
|
||||
expect(f.dispatch.mock.calls.length).toBeLessThanOrEqual(2);
|
||||
await f.settle();
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ companyId: "company-a" });
|
||||
for (const [scope] of f.dispatch.mock.calls.slice(1)) expect(scope).toEqual({ companyId: "company-a" });
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const count = f.dispatch.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.dispatch).toHaveBeenCalledTimes(count);
|
||||
});
|
||||
it("remembers the first producer arriving during drain without owner SQL", async () => {
|
||||
const f = fixture(); await f.worker.ready; f.state.draining = true;
|
||||
await f.notify(); await f.settle(); await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).toHaveBeenCalledOnce();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(2); expect(f.dispatch).toHaveBeenCalledTimes(2);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("releases a failed producer hold during drain and retains recovery for resume", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
f.state.fail = true; await f.notify(); await f.settle();
|
||||
expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
f.state.draining = true; await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(2); expect(f.next).toHaveBeenCalledOnce();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.fail = false; f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("preserves a notification during a nonoverlapping pass", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>(resolve => { release = resolve; });
|
||||
f.promote.mockImplementationOnce(() => held);
|
||||
await f.notify(); await f.settle();
|
||||
await f.notify(); await f.settle();
|
||||
expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
release(); await f.settle();
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("does not dispatch on warm standby and cancels deadlines at shutdown", async () => {
|
||||
const f = fixture({ due: 1_010_000 }); f.state.standby = true; await f.worker.ready;
|
||||
await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).not.toHaveBeenCalled();
|
||||
f.state.standby = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledOnce(); expect(f.state.due).toBeNull();
|
||||
f.state.due = Date.now() + 60_000; await f.notify(); await f.settle(); await f.worker.stop();
|
||||
await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,238 @@
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, costEvents, createDb, databaseWorkPending, heartbeatRuns } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { agentService } from "../services/agents.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_DISPATCH_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("run dispatch production wiring (controlled local process adapter)", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let worker: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]> | undefined;
|
||||
let reads: ReturnType<typeof vi.spyOn>;
|
||||
let draining = false;
|
||||
const errors: unknown[] = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("run-dispatch-work-");
|
||||
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`TRUNCATE companies CASCADE`);
|
||||
heartbeat = heartbeatService(db); draining = false; errors.length = 0;
|
||||
reads = vi.spyOn(db, "select");
|
||||
});
|
||||
afterEach(async () => {
|
||||
await worker?.stop(); await coordinator?.stop();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
worker = undefined; coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining,
|
||||
canReconcile: () => true, onError: error => errors.push(error) });
|
||||
worker = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
|
||||
await worker.ready;
|
||||
}
|
||||
async function fixture() {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Dispatch", issuePrefix: `D${companyId.slice(0, 7)}`,
|
||||
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "dispatch-user" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Controlled process", role: "engineer", status: "idle",
|
||||
adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('dispatch-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
|
||||
}).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "failed",
|
||||
responsibleUserId: "dispatch-user", errorCode: "codex_transient_upstream", finishedAt: new Date(),
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, contextSnapshot: {},
|
||||
}).returning();
|
||||
return { companyId, agentId: agent.id, runId: run.id };
|
||||
}
|
||||
async function quiet() {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull(), { timeout: 5_000 });
|
||||
const count = reads.mock.calls.length;
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
} finally { vi.useRealTimers(); }
|
||||
}
|
||||
async function status(id: string) {
|
||||
return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]);
|
||||
}
|
||||
async function completed(id: string) {
|
||||
await vi.waitFor(async () => expect(["running", "succeeded"]).toContain((await status(id))?.status), { timeout: 10_000 });
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const completed = await status(id);
|
||||
expect(completed.status).toBe("succeeded");
|
||||
expect(completed.stdoutExcerpt).toContain("dispatch-proof");
|
||||
}
|
||||
it("keeps fresh empty, configured idle and retained terminal history quiet", async () => {
|
||||
await start(); await quiet();
|
||||
await fixture(); await worker!.stop(); await coordinator!.stop();
|
||||
await start(); await quiet();
|
||||
});
|
||||
it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 4_000, random: () => 0 });
|
||||
expect(result.outcome).toBe("scheduled");
|
||||
if (result.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime()), { timeout: 5_000 });
|
||||
expect((await status(result.run.id)).status).toBe("scheduled_retry");
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(Date.now()).toBeLessThan(result.dueAt.getTime());
|
||||
expect((await status(result.run.id)).status).toBe("scheduled_retry");
|
||||
await completed(result.run.id);
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(1);
|
||||
await quiet();
|
||||
});
|
||||
it("reconstructs an overdue retry after process restart without a periodic tick", async () => {
|
||||
const f = await fixture();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 120_000), delayMs: 60_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
heartbeat = heartbeatService(db);
|
||||
await start(); await completed(scheduled.run.id); await quiet();
|
||||
});
|
||||
it("cancels a future retry and removes its old deadline", async () => {
|
||||
const f = await fixture(); await start();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime()), { timeout: 5_000 });
|
||||
await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true });
|
||||
await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled");
|
||||
});
|
||||
it("retains the first retry written during drain and catches it up after resume", async () => {
|
||||
const f = await fixture(); await start(); await quiet(); draining = true;
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 200 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
const count = reads.mock.calls.length;
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
} finally { vi.useRealTimers(); }
|
||||
expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
|
||||
draining = false; worker!.wake();
|
||||
await completed(scheduled.run.id); await quiet();
|
||||
});
|
||||
it("uses a committed budget raise to resume a queued run in the company it paused", async () => {
|
||||
const f = await fixture();
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 100,
|
||||
metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
|
||||
const [cost] = await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
provider: "fixture", biller: "fixture", model: "controlled", inputTokens: 0, outputTokens: 0,
|
||||
costCents: 125, occurredAt: new Date() }).returning();
|
||||
await budgets.evaluateCostEvent(cost);
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0].status).toBe("paused");
|
||||
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
|
||||
await start(); await quiet(); expect((await status(queued.id)).status).toBe("queued");
|
||||
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 10_000,
|
||||
metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
|
||||
await completed(queued.id); await quiet();
|
||||
});
|
||||
it("clearing an agent error wakes saved queue work only after its own transaction commits", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
await db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId));
|
||||
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
|
||||
let writing: ReturnType<ReturnType<typeof agentService>["clearError"]> | undefined;
|
||||
try {
|
||||
await db.transaction(async blocker => {
|
||||
await blocker.select().from(agents).where(eq(agents.id, f.agentId)).for("update");
|
||||
writing = agentService(db).clearError(f.agentId);
|
||||
void writing.catch(() => {});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(false);
|
||||
expect((await status(queued.id)).status).toBe("queued");
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
|
||||
});
|
||||
} finally { await writing; }
|
||||
await completed(queued.id); await quiet();
|
||||
});
|
||||
it("leaves cancellation-owned retries quiet even when their old promotion time is overdue", async () => {
|
||||
const f = await fixture();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 60_000), delayMs: 1_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await db.update(heartbeatRuns).set({ resultJson: { startupCancellation: { requestedAt: new Date().toISOString() },
|
||||
cancellation: { expected: true, reason: "operator stop", source: "operator" } } }).where(eq(heartbeatRuns.id, scheduled.run.id));
|
||||
await start(); await quiet();
|
||||
expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
|
||||
expect(await heartbeat.promoteDueScheduledRetries()).toEqual({ promoted: 0, runIds: [] });
|
||||
await quiet();
|
||||
});
|
||||
it("rejects an error clear without publishing work when the lifecycle is no longer ready", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
await db.update(agents).set({ status: "error", lifecycleState: "paused" }).where(eq(agents.id, f.agentId));
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await expect(agentService(db).clearError(f.agentId)).rejects.toThrow("Only agents in error status");
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
const count = reads.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
} finally { vi.useRealTimers(); }
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
|
||||
await quiet();
|
||||
});
|
||||
it("keeps a foreign company's future retry intact while concurrent workers claim one due retry", async () => {
|
||||
const selected = await fixture(), foreign = await fixture(); await start();
|
||||
const later = await heartbeat.scheduleBoundedRetry(foreign.runId, { delayMs: 600_000 });
|
||||
const due = await heartbeat.scheduleBoundedRetry(selected.runId, { delayMs: 500 });
|
||||
if (later.outcome !== "scheduled" || due.outcome !== "scheduled") throw new Error("Expected retries");
|
||||
const other = heartbeatService(db);
|
||||
const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const competingWorker = other.registerRunDispatchWork(competing, () => true);
|
||||
try {
|
||||
await competingWorker.ready; await completed(due.run.id); await other.drainActiveRunExecutions();
|
||||
expect((await status(later.run.id)).status).toBe("scheduled_retry");
|
||||
expect((await status(later.run.id)).scheduledRetryAt).toEqual(later.dueAt);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, selected.runId))).toHaveLength(1);
|
||||
await heartbeat.cancelRun(later.run.id, "fixture teardown", { suppressImmediateRecovery: true });
|
||||
await quiet();
|
||||
} finally { await competingWorker.stop(); await competing.stop(); await other.drainActiveRunExecutions(); }
|
||||
});
|
||||
it("holds nested producer work until outer commit and rolls back without a successor", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
|
||||
let queuedId = "";
|
||||
const writing = db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await notifyRunDispatch(nested);
|
||||
const [row] = await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
responsibleUserId: "dispatch-user", status: "queued", contextSnapshot: {} }).returning();
|
||||
queuedId = row.id;
|
||||
});
|
||||
await held;
|
||||
});
|
||||
await vi.waitFor(() => expect(queuedId).not.toBe(""));
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(await status(queuedId)).toBeUndefined();
|
||||
release(); await writing; await completed(queuedId); await quiet();
|
||||
await expect(db.transaction(async tx => {
|
||||
await tx.transaction(async nested => { await notifyRunDispatch(nested); await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "queued" }); });
|
||||
throw new Error("rollback outer");
|
||||
})).rejects.toThrow("rollback outer");
|
||||
await quiet();
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -262,6 +262,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
const [delivered] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
expect(delivered.runId).toBeTruthy();
|
||||
// The receipt can commit before execution enters liveRunExecutions.
|
||||
// Observe actual adapter admission before awaiting its drain.
|
||||
await vi.waitFor(async () => {
|
||||
const current = await heartbeat.getRun(delivered.runId!);
|
||||
expect(countExecuteCallsForRun(delivered.runId!), JSON.stringify({
|
||||
status: current?.status, errorCode: current?.errorCode,
|
||||
})).toBe(1);
|
||||
}, { timeout: 5_000 });
|
||||
await heartbeat.waitForRunExecutionDrain(delivered.runId!);
|
||||
expect(countExecuteCallsForRun(delivered.runId!)).toBe(1);
|
||||
expect((await heartbeat.getRun(delivered.runId!))?.contextSnapshot).toMatchObject({ wakeCommentIds: [comment.id] });
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared";
|
||||
import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { prepareHeartbeatTaskContext } from "../services/heartbeat/task-context.js";
|
||||
import { documentService } from "../services/documents.js";
|
||||
import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js";
|
||||
import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js";
|
||||
@@ -19,6 +20,7 @@ if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${s
|
||||
describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
const adapterType = "task_context_test";
|
||||
const captured = new Map<string, AdapterExecutionContext>();
|
||||
const inspectPreparation = new Map<string, () => void>();
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
@@ -33,6 +35,7 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
type: adapterType,
|
||||
supportsLocalAgentJwt: false,
|
||||
execute: async (input) => {
|
||||
inspectPreparation.get(input.agent.id)?.();
|
||||
captured.set(input.runId, input);
|
||||
return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} };
|
||||
},
|
||||
@@ -118,6 +121,101 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reads selected comments once for both wake projections", async () => {
|
||||
const f = await seed();
|
||||
const commentId = randomUUID();
|
||||
await db.insert(issueComments).values({
|
||||
id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user",
|
||||
body: "selected wake canary",
|
||||
});
|
||||
const select = vi.spyOn(db, "select");
|
||||
inspectPreparation.set(f.agentId, () => {
|
||||
const commentReads = select.mock.calls.filter(([projection]) => projection
|
||||
&& projection.body === issueComments.body && projection.id === issueComments.id && projection.deletedAt === issueComments.deletedAt);
|
||||
expect(commentReads).toHaveLength(1);
|
||||
});
|
||||
try {
|
||||
const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] });
|
||||
expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "selected wake canary" });
|
||||
expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "selected wake canary" }] });
|
||||
} finally {
|
||||
inspectPreparation.delete(f.agentId);
|
||||
select.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels deleted-only queued comments before adapter dispatch", async () => {
|
||||
const f = await seed(), commentId = randomUUID();
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "discarded wake", deletedAt: new Date() });
|
||||
const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented",
|
||||
payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] } });
|
||||
expect(run).not.toBeNull();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const finished = await heartbeat.getRun(run!.id);
|
||||
expect(finished).toMatchObject({ status: "cancelled", error: "Queued messages were discarded before dispatch" });
|
||||
expect(captured.has(run!.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves deleted comment metadata without delivering its body", async () => {
|
||||
const f = await seed();
|
||||
const commentId = randomUUID();
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "deleted wake canary", deletedAt: new Date(),
|
||||
presentation: { fixture: "deleted" }, metadata: { fixture: "deleted" } });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
|
||||
await prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
|
||||
policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
|
||||
});
|
||||
expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "", presentation: null, metadata: null });
|
||||
expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "", presentation: null, metadata: null }] });
|
||||
expect(context.paperclipTaskMarkdown).not.toContain("deleted wake canary");
|
||||
});
|
||||
|
||||
it.each([false, true])("uses admitted skill-test scope when recovery is %s", async (recovering) => {
|
||||
const f = await seed();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } };
|
||||
const pinned = vi.fn(async () => null);
|
||||
const preparing = prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
|
||||
policy: { recovering, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: pinned },
|
||||
});
|
||||
if (recovering) await expect(preparing).rejects.toThrow("skill test's saved version is unavailable");
|
||||
else await preparing;
|
||||
expect(pinned).toHaveBeenCalledTimes(recovering ? 1 : 0);
|
||||
expect(context.paperclipAgentJwtScope).toEqual({ kind: recovering ? "skill_test" : "standard", ...(recovering ? { issueId: f.issueId } : {}) });
|
||||
});
|
||||
|
||||
it("retains the context issue scope when no issue reference is available", async () => {
|
||||
const f = await seed();
|
||||
const otherIssueId = randomUUID(), commentId = randomUUID();
|
||||
await db.insert(issues).values({ id: otherIssueId, companyId: f.companyId, title: "Other issue", issueNumber: 2 });
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: otherIssueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "other issue canary" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
|
||||
await prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef: null, issueContext: null, context, taskSession: null },
|
||||
policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
|
||||
});
|
||||
expect(JSON.stringify(context.paperclipWake)).not.toContain("other issue canary");
|
||||
expect(context.paperclipWakeComment).toBeUndefined();
|
||||
});
|
||||
|
||||
it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => {
|
||||
const f = await seed();
|
||||
const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId });
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db";
|
||||
import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
|
||||
@@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
|
||||
(support.supported ? describe : describe.skip)("historical cleanup verification", () => {
|
||||
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true);
|
||||
const state = async () => (await read()).backgroundWork;
|
||||
@@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
|
||||
inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }),
|
||||
});
|
||||
}
|
||||
it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => {
|
||||
const f = await seed(adHoc);
|
||||
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
|
||||
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
|
||||
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value);
|
||||
let absent = false;
|
||||
const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => {
|
||||
if (!absent) return Response.json({ absent: false });
|
||||
const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000);
|
||||
expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id,
|
||||
leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId });
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url");
|
||||
const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1",
|
||||
sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url");
|
||||
return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` });
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchImpl);
|
||||
const runtime = environmentRuntimeService(db);
|
||||
const service = historicalCleanupVerificationService(db, {
|
||||
inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined,
|
||||
});
|
||||
expect(await state()).toBe("present");
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 });
|
||||
expect(await state()).toBe("present");
|
||||
absent = true;
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
|
||||
expect(await state()).toBe("none");
|
||||
const [current] = await db.select().from(environmentLeases);
|
||||
expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId });
|
||||
expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination");
|
||||
expect(await service.sweep()).toMatchObject({ checked: 0 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => {
|
||||
await seed(true);
|
||||
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
|
||||
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value);
|
||||
const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl);
|
||||
const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
|
||||
acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null,
|
||||
inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }),
|
||||
}] });
|
||||
const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined });
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 });
|
||||
expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => {
|
||||
const f = await seed(adHoc); expect(await state()).toBe("present");
|
||||
const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
approvals,
|
||||
companies,
|
||||
createDb,
|
||||
applyPendingMigrations,
|
||||
heartbeatRuns,
|
||||
issueApprovals,
|
||||
issueRelations,
|
||||
@@ -21,7 +22,10 @@ import {
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { buildIssueGraphLivenessIncidentKey } from "../services/recovery/origins.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_BLOCKER_ATTENTION_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
@@ -36,8 +40,9 @@ describeEmbeddedPostgres("issue blocker attention", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (externalUrl) await applyPendingMigrations(externalUrl);
|
||||
else tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-");
|
||||
db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
svc = issueService(db);
|
||||
}, 20_000);
|
||||
|
||||
@@ -55,6 +60,7 @@ describeEmbeddedPostgres("issue blocker attention", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
|
||||
import { registerIssueRecoveryWork } from "../services/recovery/work.js";
|
||||
import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
|
||||
import { taskWatchdogService } from "../services/task-watchdogs.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("issue recovery production producers and deadlines", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
let worker: ReturnType<ReturnType<typeof heartbeatService>["registerIssueStateRecoveryWork"]>;
|
||||
let dispatch: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]>;
|
||||
let cancellation: ReturnType<ReturnType<typeof heartbeatService>["registerScheduledRetryCancellationWork"]>;
|
||||
let draining = false;
|
||||
const tasks = new Map<string, { nextRunAt?: () => number | null; hasPending: () => Promise<boolean> }>();
|
||||
const errors: unknown[] = [];
|
||||
const passes = new Set<Promise<unknown>>();
|
||||
const extraStops: Array<() => Promise<void>> = [];
|
||||
const extraDeadlines: Array<() => number | null> = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-");
|
||||
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`TRUNCATE companies CASCADE`);
|
||||
heartbeat = heartbeatService(db); draining = false; errors.length = 0;
|
||||
});
|
||||
afterEach(async () => {
|
||||
for (const stop of extraStops.splice(0)) await stop();
|
||||
extraDeadlines.length = 0;
|
||||
await worker?.stop(); await dispatch?.stop(); await cancellation?.stop(); await coordinator?.stop();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
tasks.clear();
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => {
|
||||
tasks.set(queue, task);
|
||||
return register(queue, { ...task, run: signal => {
|
||||
const work = task.run(signal); passes.add(work);
|
||||
void work.then(() => passes.delete(work), () => passes.delete(work));
|
||||
return work;
|
||||
} });
|
||||
};
|
||||
worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining);
|
||||
dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
|
||||
cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining);
|
||||
await Promise.all([worker.ready, dispatch.ready, cancellation.ready]);
|
||||
}
|
||||
async function compete() {
|
||||
const other = heartbeatService(db);
|
||||
const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const work = other.registerIssueStateRecoveryWork(competing, () => true);
|
||||
extraStops.push(() => competing.stop()); extraDeadlines.push(competing.nextWakeAt); await work.ready;
|
||||
}
|
||||
async function fixture() {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: "recovery-user", name: "Recovery operator", email: "recovery-user@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 7)}`,
|
||||
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "recovery-user" });
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: "recovery-user", principalType: "user", status: "active", membershipRole: "owner" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Recovery process", role: "engineer", status: "idle",
|
||||
adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('recovery-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
|
||||
}).returning();
|
||||
return { companyId, agentId: agent.id };
|
||||
}
|
||||
async function settled() {
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const wake = coordinator.nextWakeAt();
|
||||
const detail = wake === null ? "" : JSON.stringify({ now: Date.now(), tasks: await Promise.all([...tasks].map(async ([queue, task]) => ({ queue, at: task.nextRunAt?.(), pending: await task.hasPending() }))),
|
||||
runs: (await db.select().from(heartbeatRuns)).map(run => ({ id: run.id, status: run.status, error: run.errorCode, lease: run.controllerLeaseExpiresAt, reason: run.contextSnapshot?.wakeReason })) });
|
||||
expect(wake, detail).toBeNull();
|
||||
expect(extraDeadlines.map(next => next())).toEqual(extraDeadlines.map(() => null));
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
}, { timeout: 40_000 });
|
||||
}
|
||||
async function quiet() {
|
||||
await settled();
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
|
||||
const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute");
|
||||
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); }
|
||||
finally { vi.useRealTimers(); reads.mockRestore(); executes.mockRestore(); }
|
||||
}
|
||||
async function finish(issueId: string) {
|
||||
await issueService(db).update(issueId, { status: "done" });
|
||||
// Closing one blocker can legitimately start its dependent. Wait for this
|
||||
// task's dispatch and execution; quiet() verifies the whole fixture later.
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(and(
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
|
||||
sql`${heartbeatRuns.status} in ('queued', 'running')`,
|
||||
))).toHaveLength(0);
|
||||
}, { timeout: 15_000 });
|
||||
const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
if (saved.length > 0) {
|
||||
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
|
||||
// Keep PostgreSQL I/O and coordinator settlement ahead of virtual time.
|
||||
// A single leap can otherwise occur before the retry deadline is armed.
|
||||
while (Date.now() <= due) {
|
||||
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
|
||||
await Promise.allSettled([...passes]);
|
||||
await immediate();
|
||||
}
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
|
||||
expect(remaining).toHaveLength(0);
|
||||
});
|
||||
for (const retry of saved) {
|
||||
const [settledRetry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retry.id));
|
||||
expect(settledRetry.status).toBe("cancelled");
|
||||
}
|
||||
}
|
||||
}
|
||||
async function completed(issueId: string) {
|
||||
let runId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const runs = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`);
|
||||
const run = runs.find(row => row.status === "running" || row.status === "succeeded");
|
||||
expect(run).toBeDefined(); runId = run!.id;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(current.status, JSON.stringify({ error: current.error, code: current.errorCode, stderr: current.stderrExcerpt })).toBe("succeeded");
|
||||
}, { timeout: 15_000 });
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.stdoutExcerpt).toContain("recovery-proof");
|
||||
return run;
|
||||
}
|
||||
it("keeps empty, configured idle, and terminal history quiet", async () => {
|
||||
await fixture(); await start(); await quiet();
|
||||
});
|
||||
it("creates a blocking child through the ordinary root service transaction", async () => {
|
||||
const f = await fixture();
|
||||
const parent = await issueService(db).create(f.companyId, { title: "Parent", status: "todo" });
|
||||
const child = await issueService(db).createChild(parent.id, { title: "Blocking child", status: "todo", blockParentUntilDone: true });
|
||||
expect(child.parentBlockerAdded).toBe(true);
|
||||
expect(await db.select().from(issueRelations).where(and(eq(issueRelations.companyId, f.companyId),
|
||||
eq(issueRelations.issueId, child.issue.id), eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1);
|
||||
});
|
||||
it.each(["startup", "committed"])("restores blocker-only and completed-action debt with no assignment candidates (%s)", async mode => {
|
||||
const f = await fixture();
|
||||
if (mode === "committed") { await start(); await quiet(); }
|
||||
const blocker = await issueService(db).create(f.companyId, { title: "Orphan blocker", status: "blocked", createdByAgentId: f.agentId });
|
||||
const dependent = await issueService(db).create(f.companyId, { title: "Blocked dependent", status: "blocked", blockedByIssueIds: [blocker.id] });
|
||||
const completedSource = await issueService(db).create(f.companyId, { title: "Completed source", status: "done" });
|
||||
const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: completedSource.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
|
||||
fingerprint: "source-terminal", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
|
||||
if (mode === "startup") await start();
|
||||
await vi.waitFor(async () => {
|
||||
expect((await db.select().from(issues).where(eq(issues.id, blocker.id)))[0].assigneeAgentId).toBe(f.agentId);
|
||||
expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
|
||||
}, { timeout: 40_000 });
|
||||
await finish(blocker.id); await finish(dependent.id); await quiet();
|
||||
});
|
||||
it("bounds subsidiary scans for100 assignment candidates and coalesces repeated company writes", async () => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
await db.update(agents).set({ lifecycleState: "preparing" }).where(eq(agents.id, f.agentId));
|
||||
await db.insert(issues).values(Array.from({ length: 100 }, (_, index) => ({ companyId: f.companyId,
|
||||
title: `Waiting setup ${index}`, status: "todo", assigneeAgentId: f.agentId })));
|
||||
const [source] = await db.insert(issues).values({ companyId: f.companyId, title: "Done source", status: "done" }).returning();
|
||||
const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: source.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
|
||||
fingerprint: "query-budget", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
|
||||
const scans = { blockers: 0, actions: 0, loads: 0 };
|
||||
const originalSelect = db.select.bind(db);
|
||||
vi.spyOn(db, "select").mockImplementation(((fields?: Record<string, unknown>) => {
|
||||
const query = originalSelect(fields as never); const from = query.from.bind(query);
|
||||
query.from = ((table: unknown) => {
|
||||
if (table === issues && fields?.id === issues.id && fields.companyId === issues.companyId && Object.keys(fields).length === 2) scans.loads++;
|
||||
if (table === issueRelations && fields?.createdByAgentId === issues.createdByAgentId) scans.blockers++;
|
||||
if (table === issueRecoveryActions && fields?.action === issueRecoveryActions) scans.actions++;
|
||||
return from(table as never);
|
||||
}) as typeof query.from;
|
||||
return query;
|
||||
}) as typeof db.select);
|
||||
await start(); await settled();
|
||||
expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(8);
|
||||
expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
|
||||
const [foreignSource] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign done", status: "done" }).returning();
|
||||
const [foreignAction] = await db.insert(issueRecoveryActions).values({ companyId: foreign.companyId, sourceIssueId: foreignSource.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "foreign-query-budget", nextAction: "Inspect" }).returning();
|
||||
const before = { ...scans };
|
||||
for (let index = 0; index < 100; index++) await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(scans.loads).toBe(before.loads);
|
||||
await settled();
|
||||
expect(scans.actions).toBe(before.actions); expect(scans.blockers).toBe(before.blockers);
|
||||
expect(scans.loads).toBeLessThanOrEqual(before.loads + 4);
|
||||
expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active");
|
||||
expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet();
|
||||
});
|
||||
it("limits an ordinary comment to its issue among one thousand open assignments", async () => {
|
||||
const f = await fixture();
|
||||
const rows = await db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({
|
||||
companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId,
|
||||
}))).returning({ id: issues.id });
|
||||
const checked: string[] = [];
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const pass = task.run(signal); passes.add(pass);
|
||||
void pass.then(() => passes.delete(pass), () => passes.delete(pass));
|
||||
return pass;
|
||||
} });
|
||||
// Observe the actual paged reconstruction boundary without executing one
|
||||
// thousand adapters. The producer, transaction, coordinator and SQL are real;
|
||||
// the other acceptance cases exercise the resulting recovery decisions.
|
||||
worker = registerIssueRecoveryWork(db, coordinator, {
|
||||
stranded: async scope => { checked.push(scope.issueId); },
|
||||
orphanBlockers: async () => {}, recoveryActions: async () => {},
|
||||
dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {},
|
||||
watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false,
|
||||
}, () => true);
|
||||
await worker.ready;
|
||||
async function settleWork() {
|
||||
for (let step = 0; step < 120; step++) {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await Promise.allSettled([...passes]); await immediate();
|
||||
if (coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return;
|
||||
}
|
||||
throw new Error("Recovery did not settle its committed work within two virtual minutes");
|
||||
}
|
||||
await settleWork();
|
||||
expect(new Set(checked).size).toBe(1000);
|
||||
checked.length = 0;
|
||||
await issueService(db).addComment(rows[0].id, "New evidence on this task", { userId: "recovery-user" });
|
||||
await settleWork();
|
||||
expect(checked).toEqual([rows[0].id]);
|
||||
// Company eligibility changes still reconstruct all assignments. Narrowing
|
||||
// comment hints must not hide unrelated work after an operator resumes it.
|
||||
checked.length = 0;
|
||||
await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
|
||||
await settleWork();
|
||||
expect(new Set(checked).size).toBe(1000);
|
||||
await quiet();
|
||||
});
|
||||
it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId });
|
||||
await completed(issue.id);
|
||||
await finish(issue.id); await quiet();
|
||||
}, 90_000);
|
||||
it("reacts to blocker resolution once and dispatches the durable dependency wake", async () => {
|
||||
const f = await fixture();
|
||||
const blocker = await issueService(db).create(f.companyId, { title: "Blocker", status: "todo" });
|
||||
const dependent = await issueService(db).create(f.companyId, { title: "Dependent", status: "blocked", assigneeAgentId: f.agentId,
|
||||
blockedByIssueIds: [blocker.id] });
|
||||
await start(); await quiet();
|
||||
await compete();
|
||||
await issueService(db).update(blocker.id, { status: "done" });
|
||||
const run = await completed(dependent.id);
|
||||
expect(run.contextSnapshot?.wakeReason).toBe("issue_blockers_resolved");
|
||||
await finish(dependent.id); await quiet();
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_blockers_resolved'`))).toHaveLength(1);
|
||||
}, 90_000);
|
||||
it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Finished subtree", status: "done" });
|
||||
await start(); await quiet();
|
||||
await compete();
|
||||
const watchdog = await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
|
||||
let reviewId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const [row] = await db.select().from(issueWatchdogs).where(eq(issueWatchdogs.id, watchdog.watchdog.id));
|
||||
expect(row.watchdogIssueId).toBeTruthy(); reviewId = row.watchdogIssueId!;
|
||||
}, { timeout: 15_000 });
|
||||
await completed(reviewId!);
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1);
|
||||
await finish(reviewId!);
|
||||
await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
}, 90_000);
|
||||
it("bounds ordinary writes to their company and reconstructs foreign stale debt after restart", async () => {
|
||||
const selected = await fixture(); await start(); await quiet();
|
||||
const foreign = await fixture();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: foreign.companyId, agentId: foreign.agentId,
|
||||
status: "failed", finishedAt: new Date() }).returning();
|
||||
const [stale] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign stale lock", status: "done",
|
||||
checkoutRunId: run.id }).returning();
|
||||
await issueService(db).create(selected.companyId, { title: "Selected change", status: "done" }); await quiet();
|
||||
expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBe(run.id);
|
||||
await worker.stop(); await dispatch.stop(); await cancellation.stop(); await coordinator.stop();
|
||||
heartbeat = heartbeatService(db); await start(); await quiet();
|
||||
expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBeNull();
|
||||
});
|
||||
it("fences nested assignment writes until outer commit and ignores rollback", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
let committedId = "";
|
||||
await db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Nested commit", status: "todo", assigneeAgentId: f.agentId }, nested);
|
||||
committedId = issue.id;
|
||||
});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
|
||||
});
|
||||
await completed(committedId); await finish(committedId); await settled();
|
||||
await expect(db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
await issueService(db).create(f.companyId, { title: "Nested rollback", status: "todo", assigneeAgentId: f.agentId }, nested);
|
||||
});
|
||||
throw new Error("rollback control");
|
||||
})).rejects.toThrow("rollback control");
|
||||
await quiet();
|
||||
expect(await db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0);
|
||||
}, 90_000);
|
||||
it("retains the first committed assignment during drain and executes on resume", async () => {
|
||||
const f = await fixture(); await start(); await quiet(); draining = true;
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Drained assignment", status: "todo", assigneeAgentId: f.agentId });
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
const reads = vi.spyOn(db, "select");
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try { await vi.advanceTimersByTimeAsync(90_000); expect(reads).not.toHaveBeenCalled(); }
|
||||
finally { vi.useRealTimers(); reads.mockRestore(); }
|
||||
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
|
||||
draining = false; worker.wake(); dispatch.wake();
|
||||
await completed(issue.id); await finish(issue.id); await quiet();
|
||||
}, 90_000);
|
||||
it("retains a fresh subtree's exact first-run grace then disarms on disable", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Fresh unassigned subtree", status: "todo" });
|
||||
await start(); await quiet();
|
||||
await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(source.createdAt.getTime() + 15_000), { timeout: 5_000 });
|
||||
expect(await db.select().from(issues).where(eq(issues.originKind, "task_watchdog"))).toHaveLength(0);
|
||||
await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
});
|
||||
it("waits for a live foreign controller's exact stored lease before clearing a terminal lock", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Remote owner", status: "done" });
|
||||
const lease = new Date(Date.now() + 5_000);
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running",
|
||||
runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: lease,
|
||||
contextSnapshot: { issueId: source.id }, processPid: 2_000_000_000, startedAt: new Date() }).returning();
|
||||
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, source.id));
|
||||
await start();
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.getTime()), { timeout: 3_000 });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running");
|
||||
await vi.waitFor(async () => expect((await db.select().from(issues).where(eq(issues.id, source.id)))[0].executionRunId).toBeNull(), { timeout: 12_000 });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded");
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
}, 90_000);
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { issueRecoveryActions, issueWatchdogs } from "@paperclipai/db";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
|
||||
import { registerIssueRecoveryWork } from "../services/recovery/work.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
const stops: Array<() => Promise<void>> = [];
|
||||
afterEach(async () => { for (const stop of stops.splice(0)) await stop(); vi.useRealTimers(); });
|
||||
function fixture() {
|
||||
vi.useFakeTimers(); vi.setSystemTime(1_000_000);
|
||||
const dialect = new PgDialect();
|
||||
const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }>, blockers: [] as Array<{ id: string; companyId: string }>, actions: [] as Array<{ id: string; companyId: string }> };
|
||||
const execute = vi.fn(async (query: any) => {
|
||||
if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
|
||||
throw new Error("unexpected SQL");
|
||||
});
|
||||
const load = vi.fn((table: unknown, distinct = false) => distinct
|
||||
? table === issueRecoveryActions ? rows.actions : rows.blockers
|
||||
: table === issueWatchdogs ? rows.watchdogs : rows.issues);
|
||||
const query = (distinct = false) => ({ from: (table: unknown) => ({ where: () => ({ orderBy: () => ({ limit: async () => load(table, distinct) }) }) }) });
|
||||
const db: any = installDatabaseWorkSignals({ execute,
|
||||
select: () => query(), selectDistinct: () => query(true),
|
||||
transaction: async (callback: (tx: any) => Promise<unknown>) => callback({ execute, select: db.select, selectDistinct: db.selectDistinct, transaction: db.transaction }),
|
||||
});
|
||||
const state = { draining: false, standby: false, suppressed: false, deadline: null as number | null, fail: false, incomplete: false };
|
||||
const check = vi.fn(async (scope: { onDeferredDeadline(at: number): void }) => {
|
||||
if (state.fail) throw new Error("database unavailable");
|
||||
if (state.deadline !== null && state.deadline > Date.now()) scope.onDeferredDeadline(state.deadline);
|
||||
});
|
||||
const orphanBlockers = vi.fn(async () => {}), recoveryActions = vi.fn(async () => {});
|
||||
const suppression = vi.fn(async () => state.suppressed);
|
||||
const owner = { orphanBlockers, recoveryActions, stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; },
|
||||
locks: check, watchdogs: async (scope: any) => { await check(scope); return { incomplete: state.incomplete ? 1 : 0 }; },
|
||||
schedulingSuppressed: suppression };
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.standby,
|
||||
canReconcile: () => !state.standby, onError: vi.fn() });
|
||||
const worker = registerIssueRecoveryWork(db, coordinator, owner, () => !state.draining && !state.standby);
|
||||
stops.push(() => coordinator.stop());
|
||||
return { db, rows, state, check, load, coordinator, worker, orphanBlockers, recoveryActions, suppression,
|
||||
notify: () => db.transaction((tx: any) => notifyIssueRecovery(tx, { companyId: "selected" })),
|
||||
settle: () => vi.advanceTimersByTimeAsync(1_000) };
|
||||
}
|
||||
describe("issue recovery obligation lifecycle", () => {
|
||||
it("has no timer, SQL, or owner probe after empty reconstruction", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull(); const loaded = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
|
||||
});
|
||||
it("reads scheduling suppression once per pass instead of once per issue", async () => {
|
||||
const f = fixture();
|
||||
f.rows.issues.push(...Array.from({ length: 25 }, (_, index) => ({ id: `issue-${index}`, companyId: "selected" })));
|
||||
await f.worker.ready;
|
||||
expect(f.check).toHaveBeenCalledTimes(75);
|
||||
expect(f.suppression).toHaveBeenCalledTimes(6);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("backs off a failed suppression read without probing an empty or failed owner repeatedly", async () => {
|
||||
const f = fixture(); f.suppression.mockRejectedValueOnce(new Error("settings unavailable"));
|
||||
await f.worker.ready; const reads = f.suppression.mock.calls.length;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
await vi.advanceTimersByTimeAsync(29_999); expect(f.suppression).toHaveBeenCalledTimes(reads);
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.suppression).toHaveBeenCalledTimes(reads + 1);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("normalizes a wide scope with cyclic SQL objects before publishing work hints", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
const wide = { companyId: "selected", endpointId: issueRecoveryActions.id, table: issueRecoveryActions };
|
||||
await f.db.transaction((tx: object) => notifyIssueRecovery(tx, wide)); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const loads = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.load).toHaveBeenCalledTimes(loads);
|
||||
});
|
||||
it("runs orphan blocker and action duties without assignment entries", async () => {
|
||||
const f = fixture(); f.rows.blockers.push({ id: "selected", companyId: "selected" });
|
||||
f.rows.actions.push({ id: "selected", companyId: "selected" }); await f.worker.ready;
|
||||
expect(f.check).not.toHaveBeenCalled();
|
||||
expect(f.orphanBlockers).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
|
||||
expect(f.recoveryActions).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("coalesces repeated company reconstruction without delaying an existing lease deadline", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
f.state.deadline = 1_015_000; await f.worker.ready;
|
||||
await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length, checked = f.check.mock.calls.length;
|
||||
for (let index = 0; index < 100; index++) { await f.notify(); await vi.advanceTimersByTimeAsync(10); }
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).toHaveBeenCalledTimes(checked);
|
||||
await vi.advanceTimersByTimeAsync(13_000);
|
||||
expect(f.state.deadline).toBe(1_015_000);
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_031_000);
|
||||
await vi.advanceTimersByTimeAsync(16_000);
|
||||
expect(f.load.mock.calls.length).toBeGreaterThan(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("retains exact grace and lease deadlines without early owner probes", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.deadline = 1_015_000;
|
||||
await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_015_000);
|
||||
const checked = f.check.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(14_999); expect(f.check).toHaveBeenCalledTimes(checked);
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.check.mock.calls.length).toBe(checked * 2);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("releases settled producer holds during domain drain while coordinator remains open", async () => {
|
||||
const f = fixture(); await f.worker.ready; f.state.draining = true;
|
||||
f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.draining = false; f.worker.wake(); await f.settle();
|
||||
expect(f.check).toHaveBeenCalled(); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("retries concrete failed entries and then disarms without reloading the company", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); f.state.fail = true;
|
||||
await f.worker.ready; const loaded = f.load.mock.calls.length;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
await vi.advanceTimersByTimeAsync(29_999); const checked = f.check.mock.calls.length;
|
||||
f.state.fail = false; await vi.advanceTimersByTimeAsync(1);
|
||||
expect(f.check.mock.calls.length).toBe(checked * 2); expect(f.load).toHaveBeenCalledTimes(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("keeps an incomplete watchdog subtree as concrete retry debt", async () => {
|
||||
const f = fixture(); f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.incomplete = true;
|
||||
await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
const loaded = f.load.mock.calls.length; f.state.incomplete = false;
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.check).toHaveBeenCalledTimes(2); expect(f.load).toHaveBeenCalledTimes(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("keeps a concurrent producer dirty until the in-flight owner has completed", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); await f.worker.ready;
|
||||
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
|
||||
f.check.mockImplementationOnce(() => held);
|
||||
await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
expect(f.check.mock.calls.length).toBeGreaterThan(3);
|
||||
});
|
||||
it("retains startup work on standby and cancels future deadlines on shutdown", async () => {
|
||||
const f = fixture(); f.state.standby = true; f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" });
|
||||
await f.worker.ready; await vi.advanceTimersByTimeAsync(90_000); expect(f.load).not.toHaveBeenCalled();
|
||||
f.state.standby = false; f.state.deadline = Date.now() + 60_000; f.worker.wake(); await f.settle();
|
||||
expect(f.check).toHaveBeenCalledOnce(); await f.worker.stop();
|
||||
await vi.advanceTimersByTimeAsync(120_000); expect(f.check).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -4151,7 +4151,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!dependencyTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-blockers-");
|
||||
db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString);
|
||||
db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
svc = issueService(db);
|
||||
await ensureIssueRelationsTable(db);
|
||||
}, 20_000);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createServer } from "node:http";
|
||||
import { createS3StorageProvider } from "../storage/s3-provider.js";
|
||||
import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js";
|
||||
import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
@@ -77,6 +79,93 @@ describe.skipIf(!support.supported)("managed skills use committed versions", ()
|
||||
.toMatchObject({ content: "complete root guide" });
|
||||
});
|
||||
|
||||
it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => {
|
||||
const companyId = await company();
|
||||
const objects = new Map<string, Buffer>();
|
||||
const server = createServer((request, response) => {
|
||||
void (async () => {
|
||||
const key = new URL(request.url!, "http://fixture").pathname;
|
||||
if (request.method === "PUT") {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) chunks.push(Buffer.from(chunk));
|
||||
let body = Buffer.concat(chunks);
|
||||
if (request.headers["content-encoding"]?.includes("aws-chunked")) {
|
||||
const decoded: Buffer[] = [];
|
||||
let offset = 0;
|
||||
while (offset < body.length) {
|
||||
const end = body.indexOf("\r\n", offset);
|
||||
const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16);
|
||||
if (size === 0) break;
|
||||
decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2;
|
||||
}
|
||||
body = Buffer.concat(decoded);
|
||||
}
|
||||
objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return;
|
||||
}
|
||||
const body = objects.get(key);
|
||||
if (!body) { response.writeHead(404).end(); return; }
|
||||
response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' });
|
||||
response.end(request.method === "HEAD" ? undefined : body);
|
||||
})().catch(() => { response.writeHead(500).end(); });
|
||||
});
|
||||
await new Promise<void>(resolve => { server.listen(0, "127.0.0.1", resolve); });
|
||||
const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address");
|
||||
const endpoint = `http://127.0.0.1:${address.port}`;
|
||||
const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" };
|
||||
const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]]));
|
||||
Object.assign(process.env, credentials);
|
||||
vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({
|
||||
bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true,
|
||||
})));
|
||||
const key = "cloud-runtime-config:v1";
|
||||
try {
|
||||
await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" });
|
||||
const run = (phase: string) => promisify(execFile)(process.execPath, [
|
||||
"--import", fileURLToPath(import.meta.resolve("tsx")),
|
||||
fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)),
|
||||
`--phase=${phase}`, `--company=${companyId}`, "--limit=50",
|
||||
], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString,
|
||||
PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture",
|
||||
PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3",
|
||||
PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1",
|
||||
PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" },
|
||||
timeout: 60_000, maxBuffer: 1024 * 1024 });
|
||||
await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration");
|
||||
expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0);
|
||||
await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } });
|
||||
const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]);
|
||||
const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const);
|
||||
const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]);
|
||||
expect([...objects.entries()]).toEqual(before);
|
||||
const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!;
|
||||
const archive = objects.get(archiveKey)!;
|
||||
const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1;
|
||||
objects.set(archiveKey, corrupted);
|
||||
const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]);
|
||||
objects.set(archiveKey, archive);
|
||||
const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!;
|
||||
const manifest = objects.get(manifestKey)!; objects.delete(manifestKey);
|
||||
const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]);
|
||||
objects.set(manifestKey, manifest);
|
||||
const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]);
|
||||
expect([...objects.entries()].sort()).toEqual([...before].sort());
|
||||
expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key)))
|
||||
.toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]);
|
||||
} finally {
|
||||
await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key));
|
||||
vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))));
|
||||
for (const [key, value] of Object.entries(previousCredentials)) {
|
||||
if (value === undefined) delete process.env[key]; else process.env[key] = value;
|
||||
}
|
||||
server.closeAllConnections(); await new Promise<void>(resolve => { server.close(() => resolve()); });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
it.each([
|
||||
{ storage: "legacy local", env: { PAPERCLIP_STORAGE_PROVIDER: "local_disk" }, expectedStatus: "local_storage" },
|
||||
{ storage: "shared S3", env: { PAPERCLIP_STORAGE_PROVIDER: "s3",
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { routineExecutionStatusService } from "../services/routine-run-status.js";
|
||||
import { attentionRetentionService } from "../services/attention-retention.js";
|
||||
import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js";
|
||||
import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js";
|
||||
import { skillVersionRetentionService } from "../services/skill-version-retention.js";
|
||||
import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js";
|
||||
@@ -135,6 +137,9 @@ const {
|
||||
})),
|
||||
reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })),
|
||||
reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })),
|
||||
registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerRetainedNativeSessionCleanupWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve(), stop: vi.fn() })),
|
||||
@@ -378,6 +383,11 @@ vi.mock("../realtime/live-events-ws.js", () => ({
|
||||
setupLiveEventsWebSocketServer: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../services/attention-retention.js", async importOriginal => ({
|
||||
...await importOriginal<typeof import("../services/attention-retention.js")>(),
|
||||
attentionRetentionService: vi.fn(() => ({ load: vi.fn(async () => ({ rows: [], complete: true })), check: vi.fn() })),
|
||||
}));
|
||||
|
||||
vi.mock("../services/join-request-retention.js", async importOriginal => ({
|
||||
...await importOriginal<typeof import("../services/join-request-retention.js")>(),
|
||||
joinRequestRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })),
|
||||
@@ -533,6 +543,13 @@ vi.mock("../services/native-runtime/native-question-bridge.js", () => ({
|
||||
validateNativeQuestionResponseInput: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../services/status-card-evaluation.js", () => ({
|
||||
statusCardEvaluationService: vi.fn(() => ({ run: vi.fn(async () => {}), next: vi.fn(async () => null), changed: vi.fn() })),
|
||||
registerStatusCardEvaluationWork: vi.fn((_coordinator, service, canRun) => ({
|
||||
ready: canRun() ? service.run(new AbortController().signal) : Promise.resolve(), wake: vi.fn(), stop: vi.fn(async () => {}),
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../services/secret-proposals.js", () => ({
|
||||
createSecretProposalsService: vi.fn(() => ({
|
||||
expiryDeadlines: vi.fn(async () => []),
|
||||
@@ -796,6 +813,28 @@ describe("startServer feedback export wiring", () => {
|
||||
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.skillVersionRetention)).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([false, true])("registers status-card evaluation only with maintenance and never ticks it from the heartbeat loop (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 1000 }));
|
||||
let tick: (() => void) | undefined;
|
||||
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => {
|
||||
if (interval === 1000) tick = callback;
|
||||
return 1 as unknown as ReturnType<typeof setInterval>;
|
||||
}) as typeof setInterval);
|
||||
try {
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(registerStatusCardEvaluationWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(statusCardEvaluationService).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
if (enabled) {
|
||||
const service = vi.mocked(statusCardEvaluationService).mock.results.at(-1)!.value;
|
||||
expect(service.run).toHaveBeenCalledOnce();
|
||||
expect(tick).toBeDefined();
|
||||
for (let i = 0; i < 3; i++) { tick!(); await settleStartupHousekeeping(); }
|
||||
expect(service.run).toHaveBeenCalledOnce();
|
||||
}
|
||||
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.statusCardEvaluation)).toEqual([]);
|
||||
} finally { timer.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each([false, true])("registers response recovery and its shutdown subscription only when maintenance is enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
@@ -929,6 +968,21 @@ describe("startServer feedback export wiring", () => {
|
||||
if (enabled) expect(heartbeatServiceMock.registerCapturedInstructionCommitWork.mock.calls[0][1]()).toBe(true);
|
||||
});
|
||||
|
||||
it.each([false, true])("registers retry and queued dispatch only when maintenance is enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled();
|
||||
if (enabled) {
|
||||
const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0];
|
||||
const app = await createAppMock.mock.results.at(-1)!.value;
|
||||
expect(coordinator).toBe(app.locals.deliveryWork);
|
||||
expect(admitted()).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])("registers retry cancellation recovery only when maintenance is enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
@@ -980,7 +1034,7 @@ describe("startServer feedback export wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])("registers archive delivery separately from retention sweeps (scheduler %s)", async enabled => {
|
||||
it.each([false, true])("registers source retention separately from archive delivery without tick scans (scheduler %s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30000 }));
|
||||
let tick: (() => void) | undefined;
|
||||
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => {
|
||||
@@ -992,12 +1046,17 @@ describe("startServer feedback export wiring", () => {
|
||||
const registration = deliveryWorkRegisterMock.mock.calls.find(([queue]) => queue === DELIVERY_QUEUES.decisionArchiveDelivery);
|
||||
if (!enabled) {
|
||||
expect(registration).toBeUndefined();
|
||||
expect(attentionRetentionService).not.toHaveBeenCalled();
|
||||
expect(decisionRetentionService).not.toHaveBeenCalled();
|
||||
expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping();
|
||||
expect(decisionRetentionService).not.toHaveBeenCalled();
|
||||
return;
|
||||
}
|
||||
expect(registration).toBeDefined();
|
||||
expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.attentionRetention)).toBe(true);
|
||||
expect(attentionRetentionService).toHaveBeenCalledOnce();
|
||||
const source = vi.mocked(attentionRetentionService).mock.results[0]!.value;
|
||||
expect(source.load).toHaveBeenCalledOnce();
|
||||
const retention = vi.mocked(decisionRetentionService).mock.results[0]!.value;
|
||||
expect(retention.deliverNotifications).toHaveBeenCalledOnce();
|
||||
expect(retention.nextArchiveNotificationAt).toHaveBeenCalledOnce();
|
||||
@@ -1005,9 +1064,10 @@ describe("startServer feedback export wiring", () => {
|
||||
const companyReads = () => vi.mocked(db.select).mock.results.filter(({ value }) =>
|
||||
vi.mocked(value.from).mock.calls.some(([table]) => table === companies)).length;
|
||||
const before = companyReads();
|
||||
expect(before).toBeGreaterThan(0); expect(tick).toBeDefined();
|
||||
expect(before).toBe(0); expect(tick).toBeDefined();
|
||||
tick!(); await settleStartupHousekeeping();
|
||||
expect(companyReads()).toBeGreaterThan(before);
|
||||
expect(companyReads()).toBe(before);
|
||||
expect(source.load).toHaveBeenCalledOnce();
|
||||
expect(retention.deliverNotifications).toHaveBeenCalledOnce();
|
||||
await registration![1].run(new AbortController().signal);
|
||||
expect(retention.deliverNotifications).toHaveBeenCalledTimes(2);
|
||||
@@ -1327,13 +1387,14 @@ describe("startServer feedback export wiring", () => {
|
||||
await startServer();
|
||||
await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
|
||||
expect(intervalCallback).not.toBeNull();
|
||||
intervalCallback?.();
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2);
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled();
|
||||
expect(retiredDetector).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
delete (runtime as Partial<typeof runtime>).reconcileProductivityReviews;
|
||||
|
||||
@@ -0,0 +1,656 @@
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { documentService } from "../services/documents.js";
|
||||
import * as companySearch from "../services/company-search.js";
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, withDedicatedDbConnection,
|
||||
heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db";
|
||||
import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "../services/status-card-evaluation.js";
|
||||
import { statusCardService } from "../services/status-cards.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARD_EVALUATION_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("status-card evaluation work", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let draining = false;
|
||||
let companyId: string, agentId: string;
|
||||
const errors: unknown[] = [];
|
||||
const evaluations = new Set<Promise<unknown>>();
|
||||
let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] });
|
||||
const actor = { agentId: null, userId: "board-user" };
|
||||
const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({
|
||||
mode, debounceSeconds: 60, intervalMinutes: 15,
|
||||
});
|
||||
// Controlled provider boundary: assignment persists a queued wake, and tests
|
||||
// explicitly run the authorized summarizer writeback. No model/provider runs.
|
||||
const wakeup = vi.fn(async (id: string, options: { payload?: Record<string, unknown> | null; reason?: string | null }) => {
|
||||
const [agent] = await db.select({ companyId: agents.companyId }).from(agents).where(eq(agents.id, id));
|
||||
return db.insert(agentWakeupRequests).values({ companyId: agent!.companyId, agentId: id, source: "assignment",
|
||||
reason: options.reason, payload: options.payload, status: "queued" }).returning();
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("status-card-evaluation-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
}, 30_000);
|
||||
beforeEach(async () => {
|
||||
companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockReset();
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] });
|
||||
await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
|
||||
await instanceSettingsService(db).updateExperimental({ enableStatusCards: true });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
await db.delete(budgetIncidents); await db.delete(budgetPolicies); await db.delete(approvals); await db.delete(costEvents);
|
||||
await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards);
|
||||
await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog);
|
||||
await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns);
|
||||
await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
async function start(service = statusCardEvaluationService(db, { wakeup })) {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
await registerStatusCardEvaluationWork(coordinator, { ...service, run: signal => {
|
||||
const work = service.run(signal);
|
||||
evaluations.add(work);
|
||||
void work.then(() => evaluations.delete(work), () => evaluations.delete(work));
|
||||
return work;
|
||||
} }, () => !draining).ready;
|
||||
}
|
||||
async function settled() {
|
||||
// Do not leap across PostgreSQL connect timers while real I/O is pending.
|
||||
// Await actual domain work, then require several settled event-loop turns
|
||||
// so a transient zero hold cannot stand in for transaction reconciliation.
|
||||
for (let turn = 0; turn < 3; turn++) {
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...evaluations]);
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await Promise.allSettled([...evaluations]);
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
}, { timeout: 5000 });
|
||||
await immediate();
|
||||
}
|
||||
}
|
||||
async function quiet() {
|
||||
await settled();
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute");
|
||||
const calls = wakeup.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(10 * 60_000);
|
||||
expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled();
|
||||
expect(wakeup).toHaveBeenCalledTimes(calls);
|
||||
reads.mockRestore(); writes.mockRestore(); executes.mockRestore();
|
||||
}
|
||||
async function writeback(cardId: string, generationIssueId: string, compile = false) {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
|
||||
await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId));
|
||||
const writer = { agentId, runId: run!.id };
|
||||
const cards = statusCardService(db);
|
||||
if (compile) await cards.writeQuery(cardId, { queries: [compiledQuery],
|
||||
title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer);
|
||||
await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer);
|
||||
await issueService(db).update(generationIssueId, { status: "done" });
|
||||
}
|
||||
async function card(mode: "manual" | "reactive" | "interval" = "reactive", refreshPolicy = policy(mode)) {
|
||||
const cards = statusCardService(db);
|
||||
const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy }, actor);
|
||||
const compile = await cards.requestCompile(created.id, actor);
|
||||
await writeback(created.id, compile.generatingIssue.id, true);
|
||||
return cards.getById(created.id).then(row => row!);
|
||||
}
|
||||
|
||||
it("settles empty, disabled, configured manual and unchanged reactive history without recurring SQL", async () => {
|
||||
await start(); await quiet();
|
||||
await card("manual"); await card("reactive"); await quiet();
|
||||
await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("ordinary issue change schedules debounce, assigns one durable wake, writes a result and becomes quiet", async () => {
|
||||
await start(); const created = await card(); await quiet();
|
||||
const changed = await issueService(db).create(companyId, { title: "New blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
|
||||
await vi.waitFor(async () => {
|
||||
expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0);
|
||||
}, { timeout: 5000 });
|
||||
const pending = await statusCardService(db).getById(created.id);
|
||||
expect(pending?.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect(pending?.nextEvalAt).toBeInstanceOf(Date);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(60_000); await settled();
|
||||
const generating = await statusCardService(db).getById(created.id);
|
||||
expect(generating?.generatingIssueId).toBeTruthy();
|
||||
const durable = await db.select().from(agentWakeupRequests);
|
||||
expect(durable).toHaveLength(1); expect(durable[0]?.payload?.issueId).toBe(generating?.generatingIssueId);
|
||||
expect(changed.companyId).toBe(companyId);
|
||||
await writeback(created.id, generating!.generatingIssueId!); await quiet();
|
||||
expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === generating?.generatingIssueId)))
|
||||
.toMatchObject([{ status: "ok", finishedAt: expect.any(Date) }]);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps card deletion scoped to its company and a missing deletion quiet", async () => {
|
||||
const removed = await card();
|
||||
await card();
|
||||
const ownCompany = companyId, ownAgent = agentId;
|
||||
companyId = randomUUID(); agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
|
||||
const other = await card();
|
||||
companyId = ownCompany; agentId = ownAgent;
|
||||
const factory = companySearch.companySearchService;
|
||||
const searchedCompanies: string[] = [];
|
||||
vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => {
|
||||
const search = factory(owner), actual = search.search;
|
||||
search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); };
|
||||
return search;
|
||||
});
|
||||
await start(); await quiet(); searchedCompanies.length = 0;
|
||||
const cards = statusCardService(db);
|
||||
expect(await cards.remove(removed.id)).toMatchObject({ id: removed.id, companyId: ownCompany });
|
||||
await settled();
|
||||
// Deleting a card only reconstructs deadlines; it is not an issue change
|
||||
// that warrants re-running any unchanged reactive card's saved search.
|
||||
expect(searchedCompanies).toEqual([]);
|
||||
expect(await cards.getById(other.id)).toMatchObject({ nextEvalAt: null, generatingIssueId: null, pendingChangeCount: 0 });
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
searchedCompanies.length = 0;
|
||||
expect(await cards.remove(removed.id)).toBeNull();
|
||||
await quiet();
|
||||
expect(searchedCompanies).toEqual([]);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("coalesces one hundred committed source writes without delaying another company's saved deadline", async () => {
|
||||
const reactive = await card();
|
||||
const originalCompany = companyId, originalAgent = agentId;
|
||||
companyId = randomUUID(); agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
|
||||
const other = await card("interval");
|
||||
const otherIssue = await issueService(db).create(companyId, { title: "Other interval blocker", status: "blocked", priority: "medium" });
|
||||
const otherReactive = await card();
|
||||
companyId = originalCompany; agentId = originalAgent;
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
const originalRun = service.run;
|
||||
let running: Promise<unknown> | undefined;
|
||||
const run = vi.spyOn(service, "run").mockImplementation(signal => {
|
||||
const result = originalRun(signal); running = result; return result;
|
||||
});
|
||||
await start(service); await settled();
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
// Give each committed producer and any actual domain work time to settle.
|
||||
// This prevents incidental overlap from hiding per-write reconstruction.
|
||||
const flush = async () => {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await running; await immediate();
|
||||
}
|
||||
};
|
||||
await flush();
|
||||
const select = vi.spyOn(db, "select");
|
||||
const factory = companySearch.companySearchService;
|
||||
const searchedCompanies: string[] = [];
|
||||
vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => {
|
||||
const search = factory(owner);
|
||||
const actual = search.search;
|
||||
search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); };
|
||||
return search;
|
||||
});
|
||||
run.mockClear();
|
||||
const burstStarted = Date.now();
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const issue = await issueService(db).create(companyId, { title: `Burst blocker ${i}`, status: "blocked", priority: "medium" });
|
||||
await flush();
|
||||
await issueService(db).addComment(issue.id, `Human update ${i}`, { userId: "board-user" });
|
||||
await flush();
|
||||
}
|
||||
await flush();
|
||||
// One eager reconciliation after idle and one bounded batched continuation;
|
||||
// the producers themselves are not counted as background SQL.
|
||||
const dueReads = () => select.mock.calls.filter(([fields]) => fields && "card" in fields && fields.card === statusCards).length;
|
||||
const nextReads = () => select.mock.calls.filter(([fields]) => fields && "at" in fields && fields.at === statusCards.nextEvalAt).length;
|
||||
const elapsedMs = Date.now() - burstStarted;
|
||||
const maximum = Math.ceil(elapsedMs / 1000) + 2;
|
||||
console.info("status-card burst", { elapsedMs, maximum, runs: run.mock.calls.length, due: dueReads(), next: nextReads(), searches: searchedCompanies.length });
|
||||
expect(run.mock.calls.length).toBeLessThanOrEqual(maximum);
|
||||
expect(dueReads()).toBeLessThanOrEqual(maximum); expect(nextReads()).toBeLessThanOrEqual(maximum);
|
||||
expect(searchedCompanies.length).toBeLessThanOrEqual(maximum);
|
||||
expect(searchedCompanies.length).toBeGreaterThan(0);
|
||||
expect(new Set(searchedCompanies)).toEqual(new Set([originalCompany]));
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
// Re-register with an existing durable deadline inside the coalescing
|
||||
// window. A company hint must not make that real deadline wait one second.
|
||||
await coordinator!.stop(); coordinator = undefined;
|
||||
const dueAt = Date.now() + 500;
|
||||
await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id));
|
||||
await start(service); await flush();
|
||||
await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" });
|
||||
await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id,
|
||||
"More changes while another company is due", { userId: "board-user" });
|
||||
await flush();
|
||||
await vi.advanceTimersByTimeAsync(500); await flush();
|
||||
await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce());
|
||||
expect(Date.now()).toBeLessThan(dueAt + 500);
|
||||
expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent);
|
||||
const generated = (await statusCardService(db).getById(other.id))!;
|
||||
expect(generated.generatingIssueId).toBeTruthy();
|
||||
expect((await statusCardService(db).getById(reactive.id))!.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect((await statusCardService(db).getById(otherReactive.id))!.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect(searchedCompanies).toContain(other.companyId);
|
||||
// Complete all ordinary results, then prove there is no recurring batch
|
||||
// timer once the requested work has settled.
|
||||
companyId = other.companyId; agentId = other.agentId!;
|
||||
await writeback(other.id, generated.generatingIssueId!);
|
||||
await statusCardService(db).update((await statusCardService(db).getById(other.id))!, { archived: true }, actor);
|
||||
companyId = originalCompany; agentId = originalAgent;
|
||||
await vi.advanceTimersByTimeAsync(120_000); await settled(); await flush();
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(reactive.id))!.generatingIssueId).toBeTruthy());
|
||||
const pending = (await statusCardService(db).getById(reactive.id))!;
|
||||
expect(pending.generatingIssueId).toBeTruthy();
|
||||
await writeback(reactive.id, pending.generatingIssueId!);
|
||||
const otherPending = (await statusCardService(db).getById(otherReactive.id))!;
|
||||
expect(otherPending.generatingIssueId).toBeTruthy();
|
||||
companyId = other.companyId; agentId = other.agentId!;
|
||||
await writeback(otherReactive.id, otherPending.generatingIssueId!);
|
||||
companyId = originalCompany; agentId = originalAgent;
|
||||
select.mockRestore();
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
}, 30_000);
|
||||
|
||||
it("bounds active-hours conversion work across twenty cards and repeated ordinary company writes", async () => {
|
||||
vi.setSystemTime(new Date("2026-11-02T18:00:00Z"));
|
||||
const hoursPolicy = { ...policy(), activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } };
|
||||
const cards = [];
|
||||
for (let i = 0; i < 20; i++) cards.push(await card("reactive", hoursPolicy));
|
||||
await start(); await quiet();
|
||||
const RealFormatter = Intl.DateTimeFormat;
|
||||
const actualParts = RealFormatter.prototype.formatToParts;
|
||||
let constructions = 0, conversions = 0;
|
||||
const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
|
||||
if (args[0] === "en-GB" && args[1]?.hourCycle === "h23") constructions++;
|
||||
return new RealFormatter(...args);
|
||||
} as unknown as typeof Intl.DateTimeFormat);
|
||||
const parts = vi.spyOn(RealFormatter.prototype, "formatToParts").mockImplementation(function (this: Intl.DateTimeFormat, date) {
|
||||
if (this.resolvedOptions().locale === "en-GB" && this.resolvedOptions().hourCycle === "h23") conversions++;
|
||||
return actualParts.call(this, date);
|
||||
});
|
||||
const target = await issueService(db).create(companyId, { title: "Closed-hours blocker", status: "blocked", priority: "medium" });
|
||||
await settled();
|
||||
expect(constructions).toBeLessThanOrEqual(1);
|
||||
expect(conversions).toBeLessThanOrEqual(1000);
|
||||
const cold = { constructions, conversions };
|
||||
constructions = 0; conversions = 0;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await issueService(db).addComment(target.id, `Closed-hours human update ${i}`, { userId: "board-user" });
|
||||
await settled();
|
||||
}
|
||||
expect(constructions).toBe(0);
|
||||
expect(conversions).toBeGreaterThanOrEqual(200);
|
||||
expect(conversions).toBeLessThanOrEqual(800);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
const opening = new Date("2026-11-03T09:00:00Z");
|
||||
for (const created of cards) expect((await statusCardService(db).getById(created.id)))
|
||||
.toMatchObject({ state: "paused_hours", pendingChangeCount: 1, nextEvalAt: opening });
|
||||
process.stdout.write(JSON.stringify({ measurement: "status-card company active-hours workload", cards: 20,
|
||||
sourceWrites: 11, cold, warm: { constructions, conversions } }) + "\n");
|
||||
constructor.mockRestore(); parts.mockRestore();
|
||||
await vi.advanceTimersByTimeAsync(opening.getTime() - Date.now()); await settled();
|
||||
expect(wakeup).toHaveBeenCalledTimes(20);
|
||||
for (const created of cards) {
|
||||
const generated = (await statusCardService(db).getById(created.id))!;
|
||||
expect(generated.generatingIssueId).toBeTruthy();
|
||||
await writeback(created.id, generated.generatingIssueId!);
|
||||
}
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
}, 60_000);
|
||||
|
||||
it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => {
|
||||
await start(); let created = await card("interval"); await settled();
|
||||
const old = coordinator!.nextWakeAt();
|
||||
created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor);
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await Promise.allSettled([...evaluations]);
|
||||
expect(coordinator!.nextWakeAt()).toBeLessThan(old!);
|
||||
});
|
||||
await statusCardService(db).update(created, { archived: true }, actor); await quiet();
|
||||
expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("restores overdue debt after the feature is disabled across its saved deadline", async () => {
|
||||
await start(); const created = await card(); await quiet();
|
||||
await issueService(db).create(companyId, { title: "Disabled blocker", status: "blocked", priority: "medium" });
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet();
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt);
|
||||
await instanceSettingsService(db).updateExperimental({ enableStatusCards: true });
|
||||
await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce());
|
||||
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); await quiet();
|
||||
});
|
||||
|
||||
it("does not react to rolled-back or nested uncommitted card configuration", async () => {
|
||||
await start(); const created = await card("manual"); await quiet();
|
||||
await expect(db.transaction(async tx => {
|
||||
await statusCardService(tx as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor);
|
||||
await vi.advanceTimersByTimeAsync(30_000); expect(wakeup).not.toHaveBeenCalled();
|
||||
throw new Error("rollback");
|
||||
})).rejects.toThrow("rollback");
|
||||
await quiet(); expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull();
|
||||
await db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await statusCardService(nested as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor);
|
||||
});
|
||||
expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull();
|
||||
});
|
||||
await settled(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("serializes competing connections when one source event preempts a future evaluation", async () => {
|
||||
const created = await card();
|
||||
await db.update(statusCards).set({ nextEvalAt: new Date(Date.now() + 3_600_000) }).where(eq(statusCards.id, created.id));
|
||||
await issueService(db).create(companyId, { title: "Concurrent blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
|
||||
const first = statusCardEvaluationService(db, { wakeup });
|
||||
const second = statusCardEvaluationService(db, { wakeup });
|
||||
first.changed(new Set([`changes:${companyId}`])); second.changed(new Set([`changes:${companyId}`]));
|
||||
await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]);
|
||||
const pending = await statusCardService(db).getById(created.id);
|
||||
expect(pending?.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect(pending?.nextEvalAt?.getTime()).toBeLessThan(Date.now() + 3_600_000);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]);
|
||||
const updated = await statusCardService(db).getById(created.id);
|
||||
expect(updated?.generatingIssueId).toBeTruthy(); expect(wakeup).toHaveBeenCalledOnce();
|
||||
expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === updated?.generatingIssueId)))
|
||||
.toHaveLength(1);
|
||||
});
|
||||
|
||||
it("finishes generation and writeback with a one-connection pool and releases dedicated locks", async () => {
|
||||
vi.useRealTimers();
|
||||
const previousDb = db;
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 1, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
try {
|
||||
const created = await card("reactive", { ...policy(), debounceSeconds: 1 });
|
||||
const countLocks = async () => Number((await db.execute<{ count: string }>(sql`
|
||||
select count(*) from pg_stat_activity where datname = current_database()
|
||||
and application_name = 'paperclip-workspace-finalization-lock'`))[0]!.count);
|
||||
const baseline = await countLocks();
|
||||
await issueService(db).create(companyId, { title: "Pool-one blocker", status: "blocked", priority: "medium" });
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
const acquired = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
|
||||
const holder = withDedicatedDbConnection(db, connection => connection.transaction(async tx => {
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`);
|
||||
acquired.resolve(); await release.promise;
|
||||
}));
|
||||
await acquired.promise;
|
||||
try {
|
||||
await service.run(new AbortController().signal);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
expect(await service.next()).toBeGreaterThan(Date.now());
|
||||
} finally { release.resolve(); await holder; }
|
||||
await service.run(new AbortController().signal);
|
||||
expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0);
|
||||
await vi.waitFor(async () => {
|
||||
await service.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
}, { timeout: 5000, interval: 100 });
|
||||
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!);
|
||||
await service.run(new AbortController().signal);
|
||||
expect(await service.next()).toBeNull();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
await vi.waitFor(async () => expect(await countLocks()).toBe(baseline));
|
||||
// Failure exits also release the physical connection and advisory lock.
|
||||
await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id));
|
||||
await service.run(new AbortController().signal);
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("retry");
|
||||
await vi.waitFor(async () => expect(await countLocks()).toBe(baseline));
|
||||
await withDedicatedDbConnection(db, connection => connection.transaction(async tx => {
|
||||
expect((await tx.execute<{ acquired: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${created.id}, 0)) as acquired`))[0]!.acquired).toBe(true);
|
||||
}));
|
||||
} finally { await db.$client.end({ timeout: 1 }); db = previousDb; }
|
||||
}, 15_000);
|
||||
|
||||
it.each([false, true])("recovers a lost committed hint before a future membership deadline (legacy=%s)", async legacy => {
|
||||
await issueService(db).create(companyId, { title: "Original blocker", status: "blocked", priority: "medium" });
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1d" });
|
||||
const created = await card();
|
||||
expect(created.nextEvalReason).toBe("membership");
|
||||
const oldExpiry = created.nextEvalAt!.getTime();
|
||||
// No coordinator exists to receive this committed source hint. A fresh
|
||||
// service has no in-memory hint and must reconstruct from durable truth.
|
||||
await issueService(db).create(companyId, { title: "Committed before crash", status: "blocked", priority: "medium" });
|
||||
if (legacy) await db.update(statusCards).set({ nextEvalReason: null }).where(eq(statusCards.id, created.id));
|
||||
const restarted = statusCardEvaluationService(db, { wakeup });
|
||||
await restarted.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBe(1);
|
||||
expect(pending.nextEvalReason).toBe("policy");
|
||||
expect(pending.nextEvalAt!.getTime()).toBeLessThan(oldExpiry);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!);
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("membership");
|
||||
});
|
||||
|
||||
it("preserves a legacy ambiguous five-minute claim without extending it on restart", async () => {
|
||||
const created = await card();
|
||||
const retry = new Date(Date.now() + 240_000);
|
||||
await db.update(statusCards).set({ nextEvalAt: retry, nextEvalReason: null }).where(eq(statusCards.id, created.id));
|
||||
await issueService(db).create(companyId, { title: "Unconsumed source", status: "blocked", priority: "medium" });
|
||||
const restarted = statusCardEvaluationService(db, { wakeup });
|
||||
await restarted.run(new AbortController().signal);
|
||||
expect(await restarted.next()).toBe(retry.getTime());
|
||||
vi.setSystemTime(retry); await restarted.run(new AbortController().signal);
|
||||
expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => {
|
||||
vi.setSystemTime(new Date("2026-11-02T18:00:00Z"));
|
||||
const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}),
|
||||
...(gate === "rate" ? { maxUpdatesPerHour: 1 } : {}), ...(gate === "budget" ? { dailyTokenCap: 100 } : {}) };
|
||||
const created = await card("reactive", refreshPolicy);
|
||||
if (gate === "rate" || gate === "budget") await db.insert(statusCardUpdates).values({ cardId: created.id, kind: "full", trigger: "reactive",
|
||||
status: "ok", startedAt: new Date(Date.now() - (gate === "rate" ? 30 * 60_000 : 60_000)), finishedAt: new Date(), inputTokens: gate === "budget" ? 100 : 0 });
|
||||
await issueService(db).create(companyId, { title: "Saved blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
await service.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBeGreaterThan(0); expect(pending.nextEvalAt!.getTime()).toBeGreaterThan(Date.now());
|
||||
const restarted = statusCardEvaluationService(db, { wakeup });
|
||||
await restarted.run(new AbortController().signal);
|
||||
expect(await restarted.next()).toBe(pending.nextEvalAt!.getTime());
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt);
|
||||
vi.setSystemTime(pending.nextEvalAt!.getTime() - 1); await restarted.run(new AbortController().signal);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("unions startup reconstruction with a scoped source change in another company", async () => {
|
||||
const firstCompany = companyId;
|
||||
const first = await card();
|
||||
await issueService(db).create(companyId, { title: "First blocker", status: "blocked", priority: "medium" });
|
||||
companyId = randomUUID(); agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `X${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
|
||||
const second = await card();
|
||||
await issueService(db).create(companyId, { title: "Second blocker", status: "blocked", priority: "medium" });
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
service.changed(new Set([`changes:${companyId}`]));
|
||||
await service.run(new AbortController().signal);
|
||||
const cards = statusCardService(db);
|
||||
expect((await cards.getById(first.id))?.companyId).toBe(firstCompany);
|
||||
expect((await cards.getById(first.id))?.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect((await cards.getById(second.id))?.pendingChangeCount).toBeGreaterThan(0);
|
||||
await vi.advanceTimersByTimeAsync(60_000); await service.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledTimes(2);
|
||||
expect(new Set(await db.select().from(agentWakeupRequests).then(rows => rows.map(row => row.companyId))))
|
||||
.toEqual(new Set([firstCompany, companyId]));
|
||||
});
|
||||
|
||||
it("keeps ignored issue activity quiet and reacts to human comments and document search membership", async () => {
|
||||
await start(); await card(); await quiet();
|
||||
await issueService(db).create(companyId, { title: "Ignored todo", status: "todo", priority: "medium" });
|
||||
await quiet(); expect(wakeup).not.toHaveBeenCalled();
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "all", q: "launch" });
|
||||
const searchCard = await card(); await quiet();
|
||||
const target = await issueService(db).create(companyId, { title: "Ordinary task", status: "todo", priority: "medium" });
|
||||
await quiet();
|
||||
await documentService(db).upsertIssueDocument({ issueId: target.id, key: "plan", format: "markdown", body: "launch evidence", createdByUserId: "board-user" });
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce();
|
||||
await writeback(searchCard.id, (await statusCardService(db).getById(searchCard.id))!.generatingIssueId!); await quiet();
|
||||
await issueService(db).addComment(target.id, "A human launch update", { userId: "board-user" });
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
});
|
||||
|
||||
it("retains relative query membership expiry across restart and emits the eventual removal", async () => {
|
||||
const now = new Date();
|
||||
const target = await issueService(db).create(companyId, { title: "Expiring blocker", status: "blocked", priority: "medium" });
|
||||
await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, target.id));
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h" });
|
||||
const created = await card();
|
||||
const expires = now.getTime() + 3_600_001;
|
||||
expect(created.nextEvalAt?.getTime()).toBe(expires);
|
||||
const restarted = statusCardEvaluationService(db, { wakeup });
|
||||
await restarted.run(new AbortController().signal);
|
||||
expect(await restarted.next()).toBe(expires);
|
||||
vi.setSystemTime(expires - 1); await restarted.run(new AbortController().signal); expect(wakeup).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(expires); await restarted.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBe(1); expect(pending.nextEvalAt!.getTime()).toBe(expires + 60_000);
|
||||
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("uses expiry of an off-page candidate to refresh offset-ranked query membership", async () => {
|
||||
const now = new Date();
|
||||
const tasks = await Promise.all((["critical", "high", "medium"] as const).map(priority => issueService(db).create(companyId, {
|
||||
title: `${priority} blocker`, priority, status: "blocked",
|
||||
})));
|
||||
await db.update(issues).set({ updatedAt: new Date(now.getTime() - 59 * 60_000) }).where(eq(issues.id, tasks[0]!.id));
|
||||
for (const task of tasks.slice(1)) await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, task.id));
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h", sort: "priority", limit: 1, offset: 1 });
|
||||
const created = await card();
|
||||
expect(Object.keys(created.fingerprint!)).toEqual([tasks[1]!.id]);
|
||||
const expires = now.getTime() + 60_001;
|
||||
expect(created.nextEvalAt?.getTime()).toBe(expires);
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
vi.setSystemTime(expires); await service.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBe(2); expect(wakeup).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
const [update] = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.generationIssueId, (await statusCardService(db).getById(created.id))!.generatingIssueId!));
|
||||
expect(new Set(update!.changes.map(change => change.issueId))).toEqual(new Set([tasks[1]!.id, tasks[2]!.id]));
|
||||
});
|
||||
|
||||
it("retains a scoped retry while another evaluator owns the card lock", async () => {
|
||||
const created = await card();
|
||||
await issueService(db).create(companyId, { title: "Locked blocker", status: "blocked", priority: "medium" });
|
||||
const acquired = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
|
||||
const holder = db.transaction(async tx => {
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`);
|
||||
acquired.resolve(); await release.promise;
|
||||
});
|
||||
await acquired.promise;
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
service.changed(new Set([`changes:${companyId}`]));
|
||||
try {
|
||||
await service.run(new AbortController().signal);
|
||||
expect(await service.next()).toBeGreaterThan(Date.now());
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull();
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
} finally { release.resolve(); await holder; }
|
||||
await vi.advanceTimersByTimeAsync(30_000); await service.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBeGreaterThan(0);
|
||||
vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("retains the durable five-minute retry after evaluation failure across restart and drain", async () => {
|
||||
const created = await card();
|
||||
await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id));
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
await service.run(new AbortController().signal);
|
||||
const failed = (await statusCardService(db).getById(created.id))!;
|
||||
const retryAt = failed.nextEvalAt!.getTime();
|
||||
expect(retryAt).toBeGreaterThan(Date.now() + 299_000); expect(wakeup).not.toHaveBeenCalled();
|
||||
await start(); expect(coordinator!.nextWakeAt()).toBe(retryAt);
|
||||
draining = true; await vi.advanceTimersByTimeAsync(5 * 60_000); await settled(); expect(wakeup).not.toHaveBeenCalled();
|
||||
// Repair the controlled malformed stored-query fault; no tested result is precreated.
|
||||
await db.update(statusCards).set({ queries: [compiledQuery] }).where(eq(statusCards.id, created.id));
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await quiet();
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull();
|
||||
});
|
||||
|
||||
it("cancels failed assignments without rerunning a completed generation and settles after retry", async () => {
|
||||
await start(); const created = await card(); await quiet();
|
||||
wakeup.mockRejectedValueOnce(new Error("assignment unavailable"));
|
||||
await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" });
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
await vi.advanceTimersByTimeAsync(60_000); await settled();
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect((await statusCardService(db).getById(created.id))!.state).toBe("error");
|
||||
});
|
||||
const failed = (await statusCardService(db).getById(created.id))!;
|
||||
expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null, nextEvalReason: null });
|
||||
expect(errors.length).toBeGreaterThan(0);
|
||||
const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id));
|
||||
const ledger = history.find(row => row.status === "failed")!;
|
||||
expect(ledger.finishedAt).toBeInstanceOf(Date);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, ledger.generationIssueId!)))[0]?.status).toBe("cancelled");
|
||||
const count = wakeup.mock.calls.length; await quiet(); expect(wakeup).toHaveBeenCalledTimes(count);
|
||||
// An explicit ordinary refresh is still the user's supported retry action.
|
||||
const retry = await statusCardService(db).requestRefresh(created.id, { trigger: "manual", actor });
|
||||
expect(retry.generatingIssue).toBeTruthy(); await writeback(created.id, retry.generatingIssue!.id); await quiet();
|
||||
expect((await statusCardService(db).getById(created.id))!.state).toBe("active");
|
||||
});
|
||||
|
||||
it("budget-owned company resume reactivates reactive work committed while paused", async () => {
|
||||
await start(); const created = await card(); await quiet();
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 100, hardStopEnabled: true, notifyEnabled: false }, "board-user");
|
||||
const [cost] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", billingType: "metered_api", model: "test",
|
||||
costCents: 200, occurredAt: new Date() }).returning();
|
||||
await budgets.evaluateCostEvent(cost!);
|
||||
expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("paused");
|
||||
await quiet();
|
||||
await issueService(db).create(companyId, { title: "Paused blocker", status: "blocked", priority: "medium" }); await quiet();
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 1000 }, "board-user");
|
||||
expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("active");
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("preserves the first write during drain until resume", async () => {
|
||||
await start(); const created = await card("manual"); await quiet();
|
||||
draining = true;
|
||||
await statusCardService(db).update(created, { refreshPolicy: policy("interval") }, actor);
|
||||
await settled(); expect(wakeup).not.toHaveBeenCalled();
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await settled();
|
||||
expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { installDatabaseWorkSignals, subscribeDatabaseWork } from "../../../packages/db/src/work-signals.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { notifyExecutionRunWork, writeExecutionControl } from "../services/execution-control-notifications.js";
|
||||
|
||||
it.each([false, true])("keeps execution queue consumers intact and emits only a known status-card company scope (%s)", async scoped => {
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: async (fn: (tx: object) => Promise<unknown>) =>
|
||||
fn({ execute: async () => [{ xid: "1" }] }) });
|
||||
const queues = [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement,
|
||||
DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors];
|
||||
const settled: Array<{ queue: string; hints?: string[] }> = [];
|
||||
const unsubscribe = [...queues, DELIVERY_QUEUES.statusCardEvaluation, DELIVERY_QUEUES.workspaceRecovery].map(queue => subscribeDatabaseWork(owner, queue, (event, hints) => {
|
||||
if (event === "settled") settled.push({ queue, hints: hints ? [...hints] : undefined });
|
||||
}));
|
||||
try {
|
||||
const scope = scoped ? { companyId: "changed-company", runId: "changed-run" } : undefined;
|
||||
await owner.transaction(async tx => {
|
||||
await notifyExecutionRunWork(tx, scope);
|
||||
expect(settled).toEqual([]);
|
||||
});
|
||||
expect(new Set(settled.map(row => row.queue))).toEqual(new Set([...queues, ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])]));
|
||||
expect(settled.find(row => row.queue === DELIVERY_QUEUES.statusCardEvaluation)?.hints)
|
||||
.toEqual(scoped ? ["changes:changed-company"] : undefined);
|
||||
settled.length = 0;
|
||||
const write = vi.fn(async () => "written");
|
||||
expect(await writeExecutionControl(owner as never, DELIVERY_QUEUES.workspaceRecovery, write, scope)).toBe("written");
|
||||
expect(write).toHaveBeenCalledOnce();
|
||||
expect(new Set(settled.map(row => row.queue))).toEqual(new Set([DELIVERY_QUEUES.issueMonitors, DELIVERY_QUEUES.workspaceRecovery,
|
||||
...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])]));
|
||||
} finally { for (const stop of unsubscribe) stop(); }
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { statusCardRefreshPolicySchema } from "@paperclipai/shared";
|
||||
import {
|
||||
chooseStatusCardUpdateKind,
|
||||
@@ -8,10 +8,12 @@ import {
|
||||
filterStatusCardChanges,
|
||||
isWithinStatusCardActiveHours,
|
||||
nextStatusCardEvaluationAt,
|
||||
nextStatusCardPolicyEligibility,
|
||||
statusCardChangesHash,
|
||||
} from "../services/status-card-update-engine.js";
|
||||
|
||||
describe("status card update engine", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
const defaultPolicy = statusCardRefreshPolicySchema.parse({ mode: "interval", intervalMinutes: 15 });
|
||||
|
||||
it("retains non-terminal and terminal status transitions plus membership changes", () => {
|
||||
@@ -78,6 +80,149 @@ describe("status card update engine", () => {
|
||||
expect(evaluateStatusCardPolicy({ policy: hours, now: new Date("2026-07-23T18:00:00.000Z"), lastChangeAt: null, updatesLastHour: 0, tokensToday: 0, manual: false }).action).toBe("pause_hours");
|
||||
});
|
||||
|
||||
it("disarms unchanged reactive evaluation while retaining exact pending eligibility", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 90, maxUpdatesPerHour: 2,
|
||||
activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } });
|
||||
const now = new Date("2026-07-23T16:59:30Z");
|
||||
expect(nextStatusCardEvaluationAt(policy, now)).toBeNull();
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 }))
|
||||
.toEqual(new Date("2026-07-24T09:00:00Z"));
|
||||
const open = new Date("2026-07-23T14:00:00Z");
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: new Date("2026-07-23T13:00:00Z"),
|
||||
completedUpdateTimes: [new Date("2026-07-23T13:30:00Z"), new Date("2026-07-23T13:45:00Z")], tokensToday: 0 }))
|
||||
.toEqual(new Date("2026-07-23T14:30:00.001Z"));
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: open, completedUpdateTimes: [], tokensToday: 100_000 }))
|
||||
.toEqual(new Date("2026-07-24T09:00:00Z"));
|
||||
});
|
||||
|
||||
it("retains empty-window pending changes without a recurring eligibility scan", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "09:00", timezone: "UTC" } });
|
||||
const now = new Date("2026-07-23T14:00:00Z");
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull();
|
||||
});
|
||||
|
||||
it("bounds empty-window formatter work before any timezone conversion", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "09:00", timezone: "Pacific/Chatham" } });
|
||||
const formatter = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function () {
|
||||
throw new Error("Empty active hours must not construct a formatter");
|
||||
});
|
||||
const now = new Date("2026-07-23T14:00:00Z");
|
||||
expect(isWithinStatusCardActiveHours(policy, now)).toBe(false);
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull();
|
||||
expect(formatter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("bounds formatter construction and reuses the next opening across cards and source writes", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "17:00", timezone: "America/St_Johns" } });
|
||||
const RealFormatter = Intl.DateTimeFormat;
|
||||
let constructions = 0;
|
||||
vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
|
||||
if (++constructions > 1) throw new Error("Card evaluation repeatedly constructed timezone formatters");
|
||||
return new RealFormatter(...args);
|
||||
} as unknown as typeof Intl.DateTimeFormat);
|
||||
const conversions = vi.spyOn(RealFormatter.prototype, "formatToParts");
|
||||
const now = new Date("2026-07-23T22:00:00Z");
|
||||
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
|
||||
const started = performance.now();
|
||||
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("2026-07-24T11:30:00Z"));
|
||||
const coldMs = performance.now() - started;
|
||||
const coldConversions = conversions.mock.calls.length;
|
||||
expect(coldConversions).toBeLessThanOrEqual(2881);
|
||||
conversions.mockClear();
|
||||
const warmStarted = performance.now();
|
||||
for (let write = 0; write < 100; write++) for (let card = 0; card < 20; card++) {
|
||||
expect(nextStatusCardPolicyEligibility({ ...input,
|
||||
policy: { ...policy, activeHours: { ...policy.activeHours! } },
|
||||
now: new Date(now.getTime() + write * 1000),
|
||||
})).toEqual(new Date("2026-07-24T11:30:00Z"));
|
||||
}
|
||||
expect(constructions).toBeLessThanOrEqual(1);
|
||||
expect(conversions.mock.calls.length).toBeLessThanOrEqual(2000);
|
||||
process.stdout.write(JSON.stringify({ measurement: "status-card active-hours cost", coldMs, coldConversions, constructions,
|
||||
warmMs: performance.now() - warmStarted, warmConversions: conversions.mock.calls.length, evaluations: 2000 }) + "\n");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["America/New_York", "02:15", "03:30", "2026-03-08T06:50:00Z", "2026-03-08T07:00:00Z"],
|
||||
["America/New_York", "02:15", "02:45", "2026-03-08T06:50:00Z", "2026-03-09T06:15:00Z"],
|
||||
["America/New_York", "02:15", "02:45", "2026-03-07T07:45:00Z", "2026-03-09T06:15:00Z"],
|
||||
["America/New_York", "01:15", "01:45", "2026-11-01T05:50:00Z", "2026-11-01T06:15:00Z"],
|
||||
["America/New_York", "23:30", "01:30", "2026-11-01T05:40:00Z", "2026-11-01T06:00:00Z"],
|
||||
["Australia/Lord_Howe", "02:15", "02:45", "2026-10-03T15:20:00Z", "2026-10-03T15:30:00Z"],
|
||||
["Australia/Lord_Howe", "01:35", "01:45", "2026-04-04T14:50:00Z", "2026-04-04T15:05:00Z"],
|
||||
["Asia/Kathmandu", "09:00", "17:00", "2026-07-23T12:00:00Z", "2026-07-24T03:15:00Z"],
|
||||
["Pacific/Apia", "00:10", "00:20", "2011-12-30T09:50:00Z", "2011-12-30T10:10:00Z"],
|
||||
])("preserves the first real opening in %s %s-%s after %s", (timezone, start, end, time, expected) => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, activeHours: { start, end, timezone } });
|
||||
const now = new Date(time);
|
||||
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
|
||||
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date(expected));
|
||||
// A later closed instant reuses the opening, while the exact opening itself
|
||||
// remains eligible without rounding it into the next minute or day.
|
||||
expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(now.getTime() + 1000) })).toEqual(new Date(expected));
|
||||
const opening = new Date(expected);
|
||||
expect(nextStatusCardPolicyEligibility({ ...input, now: opening })).toEqual(opening);
|
||||
expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(opening.getTime() + 1) })).toEqual(new Date(opening.getTime() + 1));
|
||||
});
|
||||
|
||||
it("invalidates an upcoming opening for clock rollback and changed policy values", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } });
|
||||
const calculate = (time: string, activeHours = policy.activeHours!) => {
|
||||
const now = new Date(time);
|
||||
return nextStatusCardPolicyEligibility({ policy: { ...policy, activeHours }, now,
|
||||
lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 });
|
||||
};
|
||||
expect(calculate("2026-07-23T18:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z"));
|
||||
expect(calculate("2026-07-23T08:00:00Z")).toEqual(new Date("2026-07-23T09:00:00Z"));
|
||||
expect(calculate("2026-07-23T17:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z"));
|
||||
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, start: "10:00" })).toEqual(new Date("2026-07-24T10:00:00Z"));
|
||||
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, end: "19:00" })).toEqual(new Date("2026-07-23T18:00:00Z"));
|
||||
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, timezone: "Asia/Tokyo" })).toEqual(new Date("2026-07-24T00:00:00Z"));
|
||||
});
|
||||
|
||||
it("checks the precise instant before a cached minute opening for historical second offsets", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "10:00", timezone: "Africa/Monrovia" } });
|
||||
const now = new Date("1970-01-01T08:00:00Z");
|
||||
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
|
||||
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("1970-01-01T09:45:00Z"));
|
||||
const inside = new Date("1970-01-01T09:44:45Z");
|
||||
expect(isWithinStatusCardActiveHours(policy, inside)).toBe(true);
|
||||
expect(nextStatusCardPolicyEligibility({ ...input, now: inside })).toEqual(inside);
|
||||
});
|
||||
|
||||
it("bounds formatter and upcoming-opening cache retention", () => {
|
||||
const now = new Date("2026-07-25T00:00:00Z");
|
||||
const timezonePolicy = (timezone: string) => statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "17:00", timezone } });
|
||||
const zones = Intl.supportedValuesOf("timeZone").slice(0, 34);
|
||||
const first = timezonePolicy(zones[0]!);
|
||||
isWithinStatusCardActiveHours(first, now);
|
||||
for (const zone of zones.slice(1)) isWithinStatusCardActiveHours(timezonePolicy(zone), now);
|
||||
const RealFormatter = Intl.DateTimeFormat;
|
||||
const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
|
||||
return new RealFormatter(...args);
|
||||
} as unknown as typeof Intl.DateTimeFormat);
|
||||
isWithinStatusCardActiveHours(first, now);
|
||||
expect(constructor).toHaveBeenCalledTimes(1);
|
||||
constructor.mockRestore();
|
||||
const time = (minute: number) => `${String(Math.floor(minute / 60)).padStart(2, "0")}:${String(minute % 60).padStart(2, "0")}`;
|
||||
const calculate = (minute: number) => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: time(minute), end: time(minute + 1), timezone: "UTC" } });
|
||||
return nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 });
|
||||
};
|
||||
calculate(1);
|
||||
for (let minute = 2; minute <= 258; minute++) calculate(minute);
|
||||
const conversions = vi.spyOn(Intl.DateTimeFormat.prototype, "formatToParts");
|
||||
expect(calculate(1)).toEqual(new Date("2026-07-25T00:01:00Z"));
|
||||
expect(conversions.mock.calls.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("selects full rebuilds for bounded drift rules and incremental otherwise", () => {
|
||||
const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false };
|
||||
expect(chooseStatusCardUpdateKind(base)).toBe("incremental");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
@@ -37,7 +38,8 @@ import {
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARDS_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
type Db = ReturnType<typeof createDb>;
|
||||
@@ -79,8 +81,8 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-");
|
||||
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -100,6 +102,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
@@ -189,6 +192,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
.send({ title: "Launch health", titlePinned: true });
|
||||
expect(patched.status).toBe(200);
|
||||
expect(patched.body).toMatchObject({ title: "Launch health", titlePinned: true });
|
||||
expect(patched.body).not.toHaveProperty("nextEvalReason");
|
||||
|
||||
const scheduled = await request(app)
|
||||
.patch(`/api/status-cards/${created.body.id}`)
|
||||
@@ -1149,6 +1153,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
});
|
||||
expect(summaryWrite.status).toBe(200);
|
||||
expect(summaryWrite.body.card).toMatchObject({ state: "active", queryVersion: 1, generatingIssueId: null });
|
||||
expect(summaryWrite.body.card).not.toHaveProperty("nextEvalReason");
|
||||
expect(summaryWrite.body.document.latestBody).toContain("**Decide:**");
|
||||
expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "full"))).toMatchObject({ status: "ok", finishedAt: expect.any(Date), inputTokens: 5200, outputTokens: 980 });
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import type { SQL } from "drizzle-orm";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
enrichWorkProductMetadataWithDiff,
|
||||
@@ -33,6 +35,18 @@ function createWorkProductRow(overrides: Partial<Record<string, unknown>> = {})
|
||||
};
|
||||
}
|
||||
|
||||
function retentionReads(insideTransaction: () => boolean) {
|
||||
return vi.fn(async (query: SQL) => {
|
||||
const statement = new PgDialect().sqlToQuery(query);
|
||||
expect(insideTransaction()).toBe(true);
|
||||
expect(statement.params).toContain("company-1");
|
||||
expect(statement.params).toContain("issue-1");
|
||||
if (statement.sql.includes("origin_kind")) return [{ id: "issue-1", originKind: null, originId: null }];
|
||||
if (statement.sql.includes("with recursive affected")) return [{ issueId: "issue-1" }];
|
||||
throw new Error(`Unexpected work-product fixture query: ${statement.sql}`);
|
||||
});
|
||||
}
|
||||
|
||||
describe("workProductService", () => {
|
||||
it("extracts runner totals and enriches work-product metadata", () => {
|
||||
const summary = workProductDiffSummaryFromEventPayload({
|
||||
@@ -158,11 +172,16 @@ describe("workProductService", () => {
|
||||
const insertValues = vi.fn(() => ({ returning: insertReturning }));
|
||||
const txInsert = vi.fn(() => ({ values: insertValues }));
|
||||
|
||||
let transactionOpen = false;
|
||||
const tx = {
|
||||
execute: retentionReads(() => transactionOpen),
|
||||
update: txUpdate,
|
||||
insert: txInsert,
|
||||
};
|
||||
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => await callback(tx));
|
||||
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => {
|
||||
transactionOpen = true;
|
||||
try { return await callback(tx); } finally { transactionOpen = false; }
|
||||
});
|
||||
|
||||
const svc = workProductService({ transaction } as any);
|
||||
const result = await svc.createForIssue("issue-1", "company-1", {
|
||||
@@ -194,11 +213,16 @@ describe("workProductService", () => {
|
||||
const updateSet = vi.fn(() => ({ where: updateWhere }));
|
||||
const txUpdate = vi.fn(() => ({ set: updateSet }));
|
||||
|
||||
let transactionOpen = false;
|
||||
const tx = {
|
||||
execute: retentionReads(() => transactionOpen),
|
||||
select: txSelect,
|
||||
update: txUpdate,
|
||||
};
|
||||
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => await callback(tx));
|
||||
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => {
|
||||
transactionOpen = true;
|
||||
try { return await callback(tx); } finally { transactionOpen = false; }
|
||||
});
|
||||
|
||||
const svc = workProductService({ transaction } as any);
|
||||
const result = await svc.update("work-product-1", {
|
||||
|
||||
+15
-135
@@ -1,5 +1,6 @@
|
||||
import { registerRoutineExecutionStatusWork } from "./services/routine-execution-status-work.js";
|
||||
import { routineExecutionStatusService } from "./services/routine-run-status.js";
|
||||
import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js";
|
||||
import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js";
|
||||
import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js";
|
||||
import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js";
|
||||
@@ -111,7 +112,6 @@ import { createApp } from "./app.js";
|
||||
import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js";
|
||||
import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js";
|
||||
import { logger } from "./middleware/logger.js";
|
||||
import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js";
|
||||
import { setStartupRecoveryPhase } from "./startup-recovery-state.js";
|
||||
import {
|
||||
StartupRefusalError,
|
||||
@@ -129,7 +129,6 @@ import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./mi
|
||||
import {
|
||||
feedbackService,
|
||||
applyManagedEnvironments,
|
||||
attentionService,
|
||||
backfillPrincipalAccessCompatibility,
|
||||
backfillLegacyToolOAuthTokens,
|
||||
bootstrapExecutionPolicyFromEnv,
|
||||
@@ -147,13 +146,12 @@ import {
|
||||
reconcileCodexLocalManagedHomesOnStartup,
|
||||
reconcilePersistedRuntimeServicesOnStartup,
|
||||
routineService,
|
||||
statusCardService,
|
||||
toolAccessService,
|
||||
workspaceOperationService,
|
||||
} from "./services/index.js";
|
||||
import { questionResponseDeliveryService } from "./services/question-response-delivery.js";
|
||||
import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js";
|
||||
import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js";
|
||||
import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "./services/status-card-evaluation.js";
|
||||
import { createSecretProposalsService } from "./services/secret-proposals.js";
|
||||
import { environmentRuntimeService } from "./services/environment-runtime.js";
|
||||
import { historicalCleanupVerificationService } from "./services/historical-cleanup-verification.js";
|
||||
@@ -975,9 +973,6 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
};
|
||||
const pluginWorkerManager = createPluginWorkerManager();
|
||||
const reportIncompleteWatchdogScans = createIncompleteTaskWatchdogScanReporter(
|
||||
(details, message) => logger.warn(details, message),
|
||||
);
|
||||
const heartbeat = config.heartbeatSchedulerEnabled
|
||||
? heartbeatService(db as any, { pluginWorkerManager })
|
||||
: null;
|
||||
@@ -1567,7 +1562,7 @@ async function startServerWithDatabaseTeardown(
|
||||
heartbeat.drainActiveRunExecutions();
|
||||
prepareHotRestartShutdown = heartbeat.prepareHotRestartShutdown;
|
||||
const routines = routineService(db as any, { pluginWorkerManager });
|
||||
const statusCards = statusCardService(db as any);
|
||||
const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat);
|
||||
const issues = issueService(db as any);
|
||||
const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, {
|
||||
wakeup: heartbeat.wakeup,
|
||||
@@ -1771,8 +1766,6 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
}
|
||||
|
||||
const promotion = await timeStartupRecovery("scheduled_retries", () => heartbeat.promoteDueScheduledRetries());
|
||||
await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns());
|
||||
const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions());
|
||||
if (
|
||||
recoveredGoalActions.enqueued > 0 ||
|
||||
@@ -1790,48 +1783,10 @@ async function startServerWithDatabaseTeardown(
|
||||
"startup session-goal recovery resumed durable agent goals",
|
||||
);
|
||||
}
|
||||
const reconciled = await heartbeat.reconcileStrandedAssignedIssues();
|
||||
if (
|
||||
promotion.promoted > 0 ||
|
||||
reconciled.assignmentDispatched > 0 ||
|
||||
reconciled.dispatchRequeued > 0 ||
|
||||
reconciled.continuationRequeued > 0 ||
|
||||
reconciled.successfulRunHandoffEscalated > 0 ||
|
||||
reconciled.successfulRunHandoffRetried > 0 ||
|
||||
reconciled.escalated > 0
|
||||
) {
|
||||
logger.warn(
|
||||
{ promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled },
|
||||
"startup heartbeat recovery changed assigned issue state",
|
||||
);
|
||||
}
|
||||
|
||||
const dependencyWakesReconciled = await heartbeat.reconcileResolvedDependencyWakes();
|
||||
if (dependencyWakesReconciled.healed > 0) {
|
||||
logger.warn(
|
||||
{ ...dependencyWakesReconciled },
|
||||
"startup dependency-wake reconciliation restored task execution paths",
|
||||
);
|
||||
}
|
||||
|
||||
const taskWatchdogsReconciled = await heartbeat.reconcileTaskWatchdogs();
|
||||
reportIncompleteWatchdogScans(taskWatchdogsReconciled, "startup");
|
||||
if (taskWatchdogsReconciled.triggered > 0) {
|
||||
logger.warn(
|
||||
{ ...taskWatchdogsReconciled },
|
||||
"startup task-watchdog reconciliation triggered watchdog work",
|
||||
);
|
||||
}
|
||||
|
||||
const scanned = await heartbeat.scanSilentActiveRuns();
|
||||
if (scanned.created > 0 || scanned.escalated > 0) {
|
||||
logger.warn({ ...scanned }, "startup active-run output watchdog created review work");
|
||||
}
|
||||
|
||||
const swept = await heartbeat.sweepStaleIssueLocks();
|
||||
if (swept.cleared > 0) {
|
||||
logger.warn({ ...swept }, "startup stale-lock sweeper cleared issue locks");
|
||||
}
|
||||
})().catch((err) => {
|
||||
logger.error({ err }, "startup heartbeat recovery failed");
|
||||
throw err;
|
||||
@@ -1840,6 +1795,12 @@ async function startServerWithDatabaseTeardown(
|
||||
await startupHeartbeatRecovery;
|
||||
}
|
||||
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready;
|
||||
|
||||
@@ -1919,23 +1880,6 @@ async function startServerWithDatabaseTeardown(
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerStoppedInstructionCollectionWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready;
|
||||
|
||||
const runRetentionSweep = async () => {
|
||||
const activeCompanies = await db.select({ id: companies.id }).from(companies).where(eq(companies.status, "active"));
|
||||
let archived = 0;
|
||||
for (const company of activeCompanies) {
|
||||
// Cursor pagination rebuilds the whole feed for every page; one
|
||||
// unscoped all-items build keeps this sweep at a single feed build
|
||||
// per company per tick.
|
||||
const page = await attentionService(db as any).list(company.id, {
|
||||
includeDismissed: true,
|
||||
excludeSources: ["approval", "join_request"],
|
||||
all: true,
|
||||
allowUnscopedAll: true,
|
||||
});
|
||||
archived += await retentionExecutor.autoArchive({ companyId: company.id, items: page.items });
|
||||
}
|
||||
return { archived };
|
||||
};
|
||||
// Startup housekeeping: provider and connection sweeps that every timer tick
|
||||
// repeats and that neither serving requests nor run recovery depends on.
|
||||
// Readiness does not wait for it. It is tracked as scheduler work, so idle
|
||||
@@ -1978,7 +1922,10 @@ async function startServerWithDatabaseTeardown(
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
|
||||
await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any),
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
|
||||
await step("retention_sweep", runRetentionSweep);
|
||||
await step("status_card_evaluation", () => registerStatusCardEvaluationWork(deliveryWork, statusCardEvaluation,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
|
||||
await step("attention_retention", () => registerAttentionRetentionWork(deliveryWork, attentionRetentionService(db as any),
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
|
||||
};
|
||||
|
||||
const createHeartbeatSchedulerTick = (scheduled: ReturnType<typeof registerScheduledWork>) => () => {
|
||||
@@ -1987,9 +1934,6 @@ async function startServerWithDatabaseTeardown(
|
||||
// captures the authoritative set of running heartbeat rows.
|
||||
trackHeartbeatSchedulerWork((async () => {
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => {
|
||||
logger.error({ err }, "decision retention sweep failed");
|
||||
}));
|
||||
const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses();
|
||||
if (sweptRuntimeStatuses > 0) {
|
||||
logger.info(
|
||||
@@ -2007,84 +1951,20 @@ async function startServerWithDatabaseTeardown(
|
||||
scheduleGitHubConnectionContinuitySweep();
|
||||
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
trackHeartbeatSchedulerWork((async () => {
|
||||
const experimental = await instanceSettingsService(db).getExperimental();
|
||||
if (experimental.enableStatusCards !== true) return;
|
||||
const result = await statusCards.tickDueStatusCards(new Date());
|
||||
await Promise.all(result.enqueued.map(async ({ cardId, generatingIssue }) => {
|
||||
try {
|
||||
await queueIssueAssignmentWakeup({
|
||||
heartbeat,
|
||||
issue: generatingIssue,
|
||||
reason: "status_card_update_assigned",
|
||||
mutation: "status_card.scheduler_update_requested",
|
||||
contextSource: "status_card_scheduler",
|
||||
requestedByActorType: "system",
|
||||
taskKey: `status-card:${cardId}`,
|
||||
rethrowOnError: true,
|
||||
});
|
||||
} catch (err) {
|
||||
await issues.update(generatingIssue.id, { status: "cancelled" });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
if (result.evaluated > 0 || result.enqueued.length > 0) {
|
||||
logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card scheduler tick complete");
|
||||
}
|
||||
})().catch((err) => {
|
||||
logger.error({ err }, "status-card scheduler tick failed");
|
||||
}));
|
||||
|
||||
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) {
|
||||
// Periodically reap orphaned runs (5-min staleness threshold) and make sure
|
||||
// persisted queued work is still being driven forward.
|
||||
runPeriodicRecovery(() => heartbeat
|
||||
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
|
||||
.then(() => heartbeat.promoteDueScheduledRetries())
|
||||
.then(async (promotion) => {
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
const reconciled = await heartbeat.reconcileStrandedAssignedIssues({ skipExternalConversationSettlement: true });
|
||||
if (
|
||||
promotion.promoted > 0 ||
|
||||
reconciled.assignmentDispatched > 0 ||
|
||||
reconciled.dispatchRequeued > 0 ||
|
||||
reconciled.continuationRequeued > 0 ||
|
||||
reconciled.successfulRunHandoffEscalated > 0 ||
|
||||
reconciled.successfulRunHandoffRetried > 0 ||
|
||||
reconciled.escalated > 0
|
||||
) {
|
||||
logger.warn(
|
||||
{ promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled },
|
||||
"periodic heartbeat recovery changed assigned issue state",
|
||||
);
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
const reconciled = await heartbeat.reconcileResolvedDependencyWakes();
|
||||
if (reconciled.healed > 0) {
|
||||
logger.warn({ ...reconciled }, "periodic dependency-wake reconciliation restored task execution paths");
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
const reconciled = await heartbeat.reconcileTaskWatchdogs();
|
||||
reportIncompleteWatchdogScans(reconciled, "periodic");
|
||||
if (reconciled.triggered > 0) {
|
||||
logger.warn({ ...reconciled }, "periodic task-watchdog reconciliation triggered watchdog work");
|
||||
}
|
||||
})
|
||||
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
|
||||
.then(async () => {
|
||||
const scanned = await heartbeat.scanSilentActiveRuns();
|
||||
if (scanned.created > 0 || scanned.escalated > 0) {
|
||||
logger.warn({ ...scanned }, "periodic active-run output watchdog created review work");
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
const swept = await heartbeat.sweepStaleIssueLocks();
|
||||
if (swept.cleared > 0) {
|
||||
logger.warn({ ...swept }, "periodic stale-lock sweeper cleared issue locks");
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "periodic heartbeat recovery failed");
|
||||
}));
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
|
||||
@@ -385,6 +386,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
|
||||
));
|
||||
}
|
||||
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
|
||||
@@ -40,7 +40,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
let companyId: string;
|
||||
const workers: ReturnType<typeof configureAgentLifecycle>[] = [];
|
||||
beforeAll(async () => {
|
||||
if (external) { await applyPendingMigrations(external); db = createDb(external); }
|
||||
if (external) { await applyPendingMigrations(external); db = createDb(external, { maxConnections: 6 }); }
|
||||
else { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); }
|
||||
}, 90_000);
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -80,22 +80,20 @@ export function agentRecords(db: Db, effects: LifecycleEffects) {
|
||||
throw conflict("Only agents in error status can have their error cleared");
|
||||
}
|
||||
|
||||
const updated = await db
|
||||
.update(agents)
|
||||
.set({
|
||||
status: "idle",
|
||||
pauseReason: null,
|
||||
pausedAt: null,
|
||||
errorReason: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready")))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const updated = await effects.transaction(db, existing.companyId, async tx => {
|
||||
const [current] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState })
|
||||
.from(agents).where(and(eq(agents.id, id), eq(agents.companyId, existing.companyId))).for("no key update");
|
||||
if (current?.status !== "error" || current.lifecycleState !== "ready") {
|
||||
throw conflict("Only agents in error status can have their error cleared");
|
||||
}
|
||||
await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle");
|
||||
const [saved] = await tx.update(agents).set({
|
||||
status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(),
|
||||
}).where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))).returning();
|
||||
if (!saved) throw conflict("Only agents in error status can have their error cleared");
|
||||
return saved;
|
||||
});
|
||||
|
||||
if (!updated) {
|
||||
throw conflict("Only agents in error status can have their error cleared");
|
||||
}
|
||||
return getById(updated.id);
|
||||
},
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
|
||||
import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js";
|
||||
import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js";
|
||||
import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js";
|
||||
@@ -511,6 +512,15 @@ export function createPostgresRunDispatchAdapter(
|
||||
return decideScheduledRetryGate(factsResult.facts, input.now);
|
||||
}
|
||||
|
||||
async function nextScheduledRetryAt(input: { cutoff: Date | null }): Promise<number | null> {
|
||||
const [row] = await db.select({ at: heartbeatRuns.scheduledRetryAt }).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`,
|
||||
input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined))
|
||||
.orderBy(asc(heartbeatRuns.scheduledRetryAt)).limit(1);
|
||||
return row?.at?.getTime() ?? null;
|
||||
}
|
||||
|
||||
async function listDueRetries(input: ListDueRetriesInput): Promise<DueRetryRun[]> {
|
||||
const rows = await db
|
||||
.select()
|
||||
@@ -518,6 +528,7 @@ export function createPostgresRunDispatchAdapter(
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`,
|
||||
lte(heartbeatRuns.scheduledRetryAt, input.now),
|
||||
input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined,
|
||||
),
|
||||
@@ -986,6 +997,7 @@ export function createPostgresRunDispatchAdapter(
|
||||
recordedCommentId && authors.get(recordedCommentId) === actorId ? [recordedCommentId] : [];
|
||||
}));
|
||||
if (commentIds.some(id => !recordedCommentIds.has(id))) return null;
|
||||
await notifyDeferredExecutionWork(tx, { companyId: run.companyId, requestId: wake.id });
|
||||
const [deferred] = await tx.update(agentWakeupRequests).set({
|
||||
status: "deferred_issue_execution", runId: null, claimedAt: null, finishedAt: null, error: null,
|
||||
payload: withQueuedCommentIdsInWakePayload({ ...payload, _paperclipWakeContext: context,
|
||||
@@ -1414,6 +1426,7 @@ export function createPostgresRunDispatchAdapter(
|
||||
|
||||
return {
|
||||
evaluateScheduledRetryGate,
|
||||
nextScheduledRetryAt,
|
||||
listDueRetries,
|
||||
cancelStaleQueuedRun,
|
||||
dispatchResolvedInteractionIfCurrent,
|
||||
|
||||
@@ -24,6 +24,7 @@ export type EvaluateScheduledRetryGateInput = {
|
||||
|
||||
/** Read-only scheduled-retry operations exposed to application use cases. */
|
||||
export interface ScheduledRetryReader {
|
||||
nextScheduledRetryAt(input: { cutoff: Date | null }): Promise<number | null>;
|
||||
evaluateScheduledRetryGate(input: EvaluateScheduledRetryGateInput): Promise<GateDecision>;
|
||||
listDueRetries(input: ListDueRetriesInput): Promise<DueRetryRun[]>;
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ function fakeReader(dueRuns: DueRetryRun[] = []): ScheduledRetryReader & { evalu
|
||||
const evaluateCalls: unknown[] = [];
|
||||
return {
|
||||
evaluateCalls,
|
||||
async nextScheduledRetryAt() { return null; },
|
||||
async evaluateScheduledRetryGate(input) {
|
||||
evaluateCalls.push(input);
|
||||
return { allowed: true };
|
||||
|
||||
@@ -67,6 +67,7 @@ export function createRunDispatch(db: Db, deps: RunDispatchDeps = {}) {
|
||||
});
|
||||
|
||||
return {
|
||||
nextScheduledRetryAt: (input: { cutoff: Date | null }) => adapter.nextScheduledRetryAt(input),
|
||||
evaluateScheduledRetryGate: createEvaluateScheduledRetryGate({ reader: adapter }),
|
||||
promoteScheduledRetry,
|
||||
promoteDueScheduledRetries: createPromoteDueScheduledRetries({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
|
||||
import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js";
|
||||
import { instanceSettingsService } from "../../../services/instance-settings.js";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
@@ -37,7 +38,8 @@ import type { TransactionScope } from "../application/ports.js";
|
||||
// a foreign-company row is invisible to a read, and a deferred-status
|
||||
// compare-and-set that affects no row leaves no other trace. The decision
|
||||
// branching itself is proven against plain facts in `domain/policy.test.ts`.
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WAKE_QUEUE_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
@@ -57,8 +59,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
|
||||
db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -78,6 +80,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
|
||||
import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
|
||||
@@ -610,6 +613,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
|
||||
},
|
||||
|
||||
async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const executionState = parseIssueExecutionState(issue.executionState);
|
||||
const wakeupRequest = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
@@ -637,6 +641,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
@@ -715,6 +720,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
@@ -1007,6 +1013,11 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
)).for("update");
|
||||
if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company");
|
||||
const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId);
|
||||
const previousComments = new Set(queuedCommentIdsFromWakePayload(priorPayload));
|
||||
const newComments = queuedCommentIdsFromWakePayload(input.mergedPayload).some(id => !previousComments.has(id));
|
||||
const newInteraction = readNonEmptyString(input.mergedPayload.interactionId);
|
||||
if (newComments || newInteraction && newInteraction !== readNonEmptyString(priorPayload.interactionId))
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId, requestId: prior.id });
|
||||
const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId);
|
||||
const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) &&
|
||||
priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and(
|
||||
@@ -1078,6 +1089,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
|
||||
async insertNewDeferredWake(scope, input) {
|
||||
const tx = requireAdmissionTx(scope, input.companyId);
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
...input.durableReceipt,
|
||||
companyId: input.companyId,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
|
||||
@@ -55,6 +56,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = {
|
||||
function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction {
|
||||
return {
|
||||
async updateCommentBody({ issueId, commentId, body, updatedAt }) {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const updated = await tx
|
||||
.update(issueComments)
|
||||
.set({ body, updatedAt })
|
||||
@@ -65,6 +67,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu
|
||||
},
|
||||
|
||||
async touchIssueUpdatedAt({ issueId, updatedAt }) {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(issues).set({ updatedAt }).where(and(eq(issues.id, issueId), eq(issues.companyId, companyId)));
|
||||
},
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../services/status-card-notifications.js";
|
||||
import { removePluginWorkspace } from "../services/plugin-workspaces.js";
|
||||
import { canActorReadWorkspaceOperation } from "../services/heartbeat-run-privacy.js";
|
||||
import { canActorReadExecutionWorkspace, executionWorkspaceReadSqlCondition } from "../services/authorization.js";
|
||||
@@ -1245,7 +1246,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
await runtimeLeases.release({ executionWorkspaceId: existing.id, force: true });
|
||||
|
||||
if (existing.mode === "shared_workspace") {
|
||||
await db
|
||||
await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, existing.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionWorkspaceId: null,
|
||||
@@ -1257,6 +1260,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
eq(issues.executionWorkspaceId, existing.id),
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js";
|
||||
import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../services/unsafe-workspace-export-notifications.js";
|
||||
@@ -16674,6 +16675,7 @@ export function issueRoutes(
|
||||
}
|
||||
// The click is durable fresh user intent, including when the message
|
||||
// predates a failed run's stop. Keep its content and original attribution.
|
||||
await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: locked.wake.id });
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: { ...readObject(locked.wake.payload), queuedCommentInterrupt: {
|
||||
actorId: actor.actorId, requestedAt: new Date().toISOString(),
|
||||
|
||||
@@ -16,6 +16,12 @@ import { accessService, heartbeatService, instanceSettingsService, issueService,
|
||||
import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js";
|
||||
import { assertCompanyAccess, getAccessibleResource, getActorInfo, hasCompanyAccess } from "./authz.js";
|
||||
|
||||
// Deadline provenance belongs to scheduling, not the public card contract.
|
||||
function publicCard<T extends { nextEvalReason?: unknown }>(card: T) {
|
||||
const { nextEvalReason: _reason, ...result } = card;
|
||||
return result;
|
||||
}
|
||||
|
||||
export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWakeupDeps } = {}) {
|
||||
const router = Router();
|
||||
const access = accessService(db);
|
||||
@@ -144,9 +150,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
assertCompanyAccess(req, companyId);
|
||||
await assertStatusCardsEnabled();
|
||||
const query = listStatusCardsQuerySchema.parse(req.query);
|
||||
res.json(await service.list(companyId, query.archived, {
|
||||
res.json((await service.list(companyId, query.archived, {
|
||||
issueReadCondition: await issueReadSqlCondition(db, req.actor),
|
||||
}));
|
||||
})).map(publicCard));
|
||||
});
|
||||
|
||||
router.post("/companies/:companyId/status-cards", validate(createStatusCardSchema), async (req, res) => {
|
||||
@@ -162,7 +168,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
try {
|
||||
const compile = await enqueueCompile(req, card.id);
|
||||
await logMutation(req, companyId, "status_card.created", card.id, { state: card.state });
|
||||
res.status(201).json(compile.card);
|
||||
res.status(201).json(publicCard(compile.card));
|
||||
} catch (error) {
|
||||
await service.remove(card.id);
|
||||
throw error;
|
||||
@@ -173,9 +179,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
await assertStatusCardsEnabled();
|
||||
const card = await getAccessibleResource(req, res, service.getById(req.params.id as string), "Status card not found");
|
||||
if (!card) return;
|
||||
res.json(await service.hydrate(card, {
|
||||
res.json(publicCard(await service.hydrate(card, {
|
||||
issueReadCondition: await issueReadSqlCondition(db, req.actor),
|
||||
}));
|
||||
})));
|
||||
});
|
||||
|
||||
router.patch("/status-cards/:id", validate(patchStatusCardSchema), async (req, res) => {
|
||||
@@ -197,7 +203,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
fields: Object.keys(req.body),
|
||||
archived: Boolean(updated.archivedAt),
|
||||
});
|
||||
res.json(compile?.card ?? restore?.card ?? updated);
|
||||
res.json(publicCard(compile?.card ?? restore?.card ?? updated));
|
||||
});
|
||||
|
||||
router.delete("/status-cards/:id", async (req, res) => {
|
||||
@@ -234,7 +240,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
generatingIssueId: result.generatingIssue.id,
|
||||
alreadyGenerating: result.alreadyGenerating,
|
||||
});
|
||||
res.status(result.alreadyGenerating ? 200 : 202).json(result);
|
||||
res.status(result.alreadyGenerating ? 200 : 202).json({ ...result, card: publicCard(result.card) });
|
||||
});
|
||||
|
||||
router.post("/status-cards/:id/refresh", validate(refreshStatusCardSchema), async (req, res) => {
|
||||
@@ -249,7 +255,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
alreadyGenerating: result.alreadyGenerating,
|
||||
enqueued: result.enqueued,
|
||||
});
|
||||
res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json(result);
|
||||
res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json({ ...result, card: publicCard(result.card) });
|
||||
});
|
||||
|
||||
router.get("/status-cards/:id/dry-run", async (req, res) => {
|
||||
@@ -289,7 +295,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
generationIssueId: req.body.generationIssueId,
|
||||
changeSummary: req.body.changeSummary,
|
||||
});
|
||||
res.json(updated);
|
||||
res.json(publicCard(updated));
|
||||
});
|
||||
|
||||
router.put("/status-cards/:id/summary", validate(writeStatusCardSummarySchema), async (req, res) => {
|
||||
@@ -309,7 +315,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
documentId: result.document.id,
|
||||
changeSummary: req.body.changeSummary,
|
||||
});
|
||||
res.json(result);
|
||||
res.json({ ...result, card: publicCard(result.card) });
|
||||
});
|
||||
|
||||
return router;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -743,8 +745,10 @@ export function accessService(db: Db) {
|
||||
sql`${issues.status} not in ('done', 'cancelled')`,
|
||||
);
|
||||
await notifyNativeStatusReassessment(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const resetInProgress = await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared";
|
||||
@@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {
|
||||
if (!reason.trim()) throw conflict("A repair reason is required");
|
||||
const before = await inspect(tx, companyId);
|
||||
if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing");
|
||||
if (before.findings.some(f => f.repairable && f.kind === "agent_projection"))
|
||||
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
|
||||
for (const finding of before.findings.filter(f => f.repairable)) {
|
||||
if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId));
|
||||
if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId)));
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
@@ -16,6 +19,7 @@ import { clearPrimaryAgent } from "./primary-agent.js";
|
||||
import { budgetServiceInTransaction } from "./budgets.js";
|
||||
|
||||
export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise<AgentConfigurationPatch> {
|
||||
await notifyAttentionRetentionWork(db, { companyId: existing.companyId });
|
||||
const id = existing.id;
|
||||
const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db);
|
||||
if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) {
|
||||
@@ -83,6 +87,8 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
|
||||
existing.adapterConfig,
|
||||
);
|
||||
}
|
||||
await notifyRunDispatch(db, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
|
||||
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
|
||||
@@ -269,6 +271,7 @@ export async function prepareConversationTurn(
|
||||
context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId;
|
||||
}
|
||||
}
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await tx
|
||||
@@ -389,6 +392,7 @@ export async function settleConversationTurn(
|
||||
issue.conversationState === conversationState
|
||||
)
|
||||
return true;
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id });
|
||||
@@ -544,6 +548,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo
|
||||
eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId),
|
||||
)).for("update");
|
||||
if (!isConversation(issue) || comment.conversationSessionGeneration != null) return;
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const released = await tx.update(issueTreeHolds).set({
|
||||
status: "released", releasedAt: new Date(), updatedAt: new Date(),
|
||||
releasedByActorType: "user", releasedByUserId: comment.authorUserId,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -34,10 +36,12 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str
|
||||
if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", {
|
||||
code: "agent_prompt_injection_accounting_pending",
|
||||
});
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
await notifyExternalConversationSettlement(tx, { companyId });
|
||||
await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id));
|
||||
await tx
|
||||
.update(issues)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
|
||||
import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared";
|
||||
import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils";
|
||||
@@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js";
|
||||
import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
|
||||
export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData> {
|
||||
await notifyAttentionRetentionWork(db, { companyId });
|
||||
const { runner, ...data } = input;
|
||||
const selection = options?.runnerResolved
|
||||
? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} }
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
|
||||
import { isAgentStatusInvokable } from "@paperclipai/shared";
|
||||
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
|
||||
@@ -38,7 +41,12 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
|
||||
},
|
||||
notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle),
|
||||
async beforeStatusChange(tx, companyId, agentId, before, after) {
|
||||
if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
if (before !== after) {
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
|
||||
await notifyNativeResponsePresentation(tx, { companyId, agentId });
|
||||
await notifyNativeReviewResponse(tx, { companyId, agentId });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { agents, projects, type Db } from "@paperclipai/db";
|
||||
import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared";
|
||||
@@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c
|
||||
const added = requested.filter(id => !previous.includes(id));
|
||||
const next = [...previous, ...added];
|
||||
if (added.length) {
|
||||
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
|
||||
await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: {
|
||||
...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next },
|
||||
} }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { type BudgetServiceHooks } from "./budgets.js";
|
||||
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
@@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
const updated = await db
|
||||
const updated = await db.transaction(async tx => {
|
||||
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
|
||||
return tx
|
||||
.update(agents)
|
||||
.set({
|
||||
permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }),
|
||||
@@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
.where(eq(agents.id, id))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
});
|
||||
|
||||
return updated ? getById(updated.id) : null;
|
||||
},
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm";
|
||||
import { agents, decisionRetention, decisions, heartbeatRuns,
|
||||
issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db";
|
||||
import { evaluateAgentInvokability } from "./agent-invokability.js";
|
||||
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
|
||||
import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js";
|
||||
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
import { attentionIssueCondition } from "./issue-visibility.js";
|
||||
import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js";
|
||||
import { issueService } from "./issues.js";
|
||||
import { parseIssueExecutionState } from "./issue-execution-policy.js";
|
||||
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
|
||||
import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
|
||||
import type { AttentionRetentionScope } from "./retention-work-notifications.js";
|
||||
|
||||
const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
|
||||
const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review",
|
||||
"failed_run", "budget_alert", "agent_error_alert"] as const;
|
||||
type Kind = typeof kinds[number];
|
||||
type Scope = AttentionRetentionScope;
|
||||
type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null };
|
||||
type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" };
|
||||
type Cursor = { sourceKind: string; sourceId: string };
|
||||
|
||||
type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number;
|
||||
blockedActivityAt: string | null; blockedDeadlineAt: number | null };
|
||||
type PageEvidence = {
|
||||
decisions: Map<string, { originIssueId: string }>;
|
||||
recoveries: Map<string, typeof issueRecoveryActions.$inferSelect>;
|
||||
runs: Map<string, typeof heartbeatRuns.$inferSelect>;
|
||||
agents: Set<string>;
|
||||
visibleIssues: Map<string, PageIssue>;
|
||||
newerRuns: Set<string>;
|
||||
deferredRoots: Set<string>;
|
||||
blockers: Awaited<ReturnType<ReturnType<typeof issueService>["listBlockerAttention"]>>;
|
||||
};
|
||||
|
||||
const sourceTables: Record<Kind, SQL> = {
|
||||
decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`,
|
||||
recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`,
|
||||
failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`,
|
||||
};
|
||||
|
||||
const sources: Record<Kind, SQL> = {
|
||||
decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id
|
||||
from decisions d where d.status = 'open'`,
|
||||
issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text
|
||||
from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`,
|
||||
recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text
|
||||
from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`,
|
||||
blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text
|
||||
from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`,
|
||||
review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text
|
||||
from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`,
|
||||
failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId')
|
||||
from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists (
|
||||
select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id
|
||||
and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`,
|
||||
budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b
|
||||
where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0
|
||||
and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`,
|
||||
agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`,
|
||||
};
|
||||
|
||||
export function attentionRetentionService(db: Db) {
|
||||
const cursors = new Map<string, Cursor>();
|
||||
async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise<Candidate[]> {
|
||||
const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []);
|
||||
const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind)
|
||||
&& (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind)));
|
||||
if (!selected.length) return [];
|
||||
const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)})
|
||||
select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId",
|
||||
s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt"
|
||||
from source_candidates s inner join companies c on c.id = s.company_id
|
||||
left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id
|
||||
where c.status = 'active'
|
||||
${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``}
|
||||
and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid)
|
||||
and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null})
|
||||
and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null))
|
||||
${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``}
|
||||
order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[];
|
||||
return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) }));
|
||||
}
|
||||
|
||||
async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) {
|
||||
const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId),
|
||||
attentionIssueCondition(conversations)));
|
||||
return row;
|
||||
}
|
||||
|
||||
async function interactionEligibility(database: Db, candidates: Candidate[]) {
|
||||
const result = new Map<string, boolean>();
|
||||
const byCompany = new Map<string, string[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.sourceKind !== "issue_thread_interaction") continue;
|
||||
const ids = byCompany.get(candidate.companyId) ?? [];
|
||||
ids.push(candidate.sourceId);
|
||||
byCompany.set(candidate.companyId, ids);
|
||||
}
|
||||
for (const [companyId, ids] of byCompany) {
|
||||
const rows = await database.select().from(issueThreadInteractions).where(and(
|
||||
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids),
|
||||
eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId)));
|
||||
if (!rows.length) continue;
|
||||
const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId),
|
||||
inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true)));
|
||||
const visibleIds = new Set(visible.map(row => row.id));
|
||||
const roster = await database.select().from(agents).where(eq(agents.companyId, companyId));
|
||||
const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId
|
||||
|| !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable;
|
||||
const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId);
|
||||
const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and(
|
||||
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds),
|
||||
eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"),
|
||||
isNull(issueThreadInteractions.addresseeUserId))) : [];
|
||||
const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id));
|
||||
for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row)
|
||||
&& (row.kind !== "request_confirmation" || newest.has(row.id)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function reviewEligibility(database: Db, candidates: Candidate[]) {
|
||||
const result = new Map<string, boolean>();
|
||||
const byCompany = new Map<string, string[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.sourceKind !== "review") continue;
|
||||
const ids = byCompany.get(candidate.companyId) ?? [];
|
||||
ids.push(candidate.sourceId);
|
||||
byCompany.set(candidate.companyId, ids);
|
||||
}
|
||||
for (const [companyId, ids] of byCompany) {
|
||||
const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId),
|
||||
inArray(issues.id, ids), attentionIssueCondition(false)));
|
||||
const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals)
|
||||
.innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and(
|
||||
eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId),
|
||||
inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending")));
|
||||
const approved = new Set(pendingApprovals.map(row => row.issueId));
|
||||
const stalled = await issueService(database).listReviewAttention(companyId, rows);
|
||||
for (const row of rows) {
|
||||
const state = parseIssueExecutionState(row.executionState);
|
||||
result.set(row.id, requiresReviewAttention({
|
||||
hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user",
|
||||
hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id),
|
||||
stalled: stalled.get(row.id)?.state === "stalled",
|
||||
}));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function pageEvidence(database: Db, candidates: Candidate[]) {
|
||||
const byCompany = new Map<string, Candidate[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue;
|
||||
const rows = byCompany.get(candidate.companyId) ?? [];
|
||||
rows.push(candidate);
|
||||
byCompany.set(candidate.companyId, rows);
|
||||
}
|
||||
const result = new Map<string, PageEvidence>();
|
||||
for (const [companyId, rows] of byCompany) {
|
||||
const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId);
|
||||
const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention");
|
||||
const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId })
|
||||
.from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : [];
|
||||
const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : [];
|
||||
const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : [];
|
||||
const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and(
|
||||
eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]),
|
||||
sql`${agents.status} <> 'terminated'`)) : [];
|
||||
const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues),
|
||||
activityAt: sql<string>`${issues.updatedAt}::text`,
|
||||
deadlineAt: sql<number>`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
|
||||
blockedActivityAt: sql<string | null>`${issues.blockedTransitionAt}::text`,
|
||||
blockedDeadlineAt: sql<number | null>`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
|
||||
}).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]),
|
||||
attentionIssueCondition(false))) : [];
|
||||
const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId),
|
||||
...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id));
|
||||
const visibleRows = await readIssues(issueIds);
|
||||
const visible = new Map(visibleRows.map(row => [row.id, row]));
|
||||
const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id));
|
||||
const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots)
|
||||
: { attention: new Map(), deferredRootIds: new Set<string>() };
|
||||
const blockers: PageEvidence["blockers"] = blockerBatch.attention;
|
||||
const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention)
|
||||
.map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id));
|
||||
for (const row of await readIssues(terminalIds)) visible.set(row.id, row);
|
||||
result.set(companyId, {
|
||||
decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])),
|
||||
runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible,
|
||||
newerRuns: await listFailedRunsWithNewerRun(database, companyId,
|
||||
runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map<string, boolean>, interactionAttention?: Map<string, boolean>, evidence?: PageEvidence): Promise<Entry | null> {
|
||||
if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) {
|
||||
return { ...candidate, rootId: candidate.sourceId, variant,
|
||||
id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() };
|
||||
}
|
||||
let sourceId = candidate.sourceId;
|
||||
let activityAt = candidate.activityAt;
|
||||
let dueAt = candidate.deadlineAt;
|
||||
if (candidate.sourceKind === "decision") {
|
||||
const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions)
|
||||
.where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0];
|
||||
if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId)
|
||||
: await visibleIssue(database, candidate.companyId, row.originIssueId))) return null;
|
||||
} else if (candidate.sourceKind === "issue_thread_interaction") {
|
||||
const attention = interactionAttention ?? await interactionEligibility(database, [candidate]);
|
||||
if (!attention.get(sourceId)) return null;
|
||||
} else if (candidate.sourceKind === "recovery_action") {
|
||||
const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0];
|
||||
if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId)
|
||||
: await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null;
|
||||
} else if (candidate.sourceKind === "review") {
|
||||
const attention = reviewAttention ?? await reviewEligibility(database, [candidate]);
|
||||
if (!attention.get(sourceId)) return null;
|
||||
} else if (candidate.sourceKind === "blocker_attention") {
|
||||
const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
|
||||
if (!row) return null;
|
||||
if (variant === "owner") {
|
||||
if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null;
|
||||
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at,
|
||||
ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
|
||||
activityAt = activity.at; dueAt = Number(activity.due);
|
||||
} else {
|
||||
const attention = evidence ? evidence.blockers.get(row.id) : (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id);
|
||||
if (!terminalBlockerRequiresAttention(attention)) return null;
|
||||
sourceId = attention.terminalBlockerIssueId ?? row.id;
|
||||
if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`);
|
||||
const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
|
||||
if (!terminal) return null;
|
||||
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at,
|
||||
ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
|
||||
activityAt = activity.at; dueAt = Number(activity.due);
|
||||
}
|
||||
const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId),
|
||||
eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId)));
|
||||
if (retention?.keep || retention?.archivedAt) return null;
|
||||
} else if (candidate.sourceKind === "failed_run") {
|
||||
const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0];
|
||||
if (!run) return null;
|
||||
const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId),
|
||||
eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0];
|
||||
const issueId = readRunIssueId(run.contextSnapshot);
|
||||
if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId)
|
||||
: await visibleIssue(database, candidate.companyId, issueId))) return null;
|
||||
if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null;
|
||||
}
|
||||
return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant,
|
||||
id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt };
|
||||
}
|
||||
|
||||
async function load(scope?: Scope, signal?: AbortSignal) {
|
||||
const key = JSON.stringify(scope ?? {});
|
||||
const rows = await candidates(db, scope, cursors.get(key));
|
||||
const entries: Entry[] = [];
|
||||
const reviews = await reviewEligibility(db, rows);
|
||||
const interactions = await interactionEligibility(db, rows);
|
||||
const evidence = await pageEvidence(db, rows);
|
||||
for (const row of rows) {
|
||||
if (signal?.aborted) return { rows: null, complete: true };
|
||||
for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) {
|
||||
const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId));
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
}
|
||||
if (rows.length === 100) {
|
||||
const last = rows[rows.length - 1]!;
|
||||
cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId });
|
||||
} else cursors.delete(key);
|
||||
return { rows: entries, complete: rows.length < 100 };
|
||||
}
|
||||
|
||||
async function check(entry: Entry) {
|
||||
return db.transaction(async transaction => {
|
||||
const database = transaction as unknown as Db;
|
||||
await database.execute(sql`select id from ${sourceTables[entry.sourceKind]}
|
||||
where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`);
|
||||
const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId);
|
||||
if (!candidate) return null;
|
||||
const current = await eligible(database, candidate, entry.variant, true);
|
||||
if (!current || current.dueAt > Date.now()) return current;
|
||||
const retention = decisionRetentionService(database);
|
||||
const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt };
|
||||
const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!;
|
||||
await retention.autoArchive({ companyId: current.companyId, items: [{ ...item,
|
||||
keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] });
|
||||
return null;
|
||||
});
|
||||
}
|
||||
return { load, check };
|
||||
}
|
||||
|
||||
export function registerAttentionRetentionWork(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
service: ReturnType<typeof attentionRetentionService>, canRun: () => boolean) {
|
||||
return registerScopedDeadlineWork(coordinator, {
|
||||
queue: DELIVERY_QUEUES.attentionRetention,
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope,
|
||||
matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId)
|
||||
&& (!scope?.sourceKind || scope.sourceKind === entry.sourceKind)
|
||||
&& (!scope?.sourceId || scope.sourceId === entry.sourceId)
|
||||
&& (!scope?.issueId || scope.issueId === entry.issueId)
|
||||
&& (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)),
|
||||
load: service.load, check: service.check,
|
||||
}, canRun);
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm";
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
|
||||
export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
|
||||
export const ATTENTION_BUDGET_WARNING_PERCENT = 85;
|
||||
export const ATTENTION_AGENT_ERROR_STATUS = "error";
|
||||
|
||||
export function collapsePendingConfirmationsToNewest<T extends {
|
||||
id: string;
|
||||
issueId: string;
|
||||
kind: string;
|
||||
createdAt: Date;
|
||||
payload: unknown;
|
||||
}>(rows: T[]) {
|
||||
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
|
||||
const newestByGroup = new Map<string, T>();
|
||||
for (const row of rows) {
|
||||
if (row.kind !== "request_confirmation" || independent(row)) continue;
|
||||
const groupKey = `${row.issueId}:${row.kind}`;
|
||||
const newest = newestByGroup.get(groupKey);
|
||||
if (
|
||||
!newest
|
||||
|| row.createdAt.getTime() > newest.createdAt.getTime()
|
||||
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
|
||||
) {
|
||||
newestByGroup.set(groupKey, row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows.filter((row) => (
|
||||
row.kind !== "request_confirmation"
|
||||
|| independent(row)
|
||||
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
|
||||
));
|
||||
}
|
||||
|
||||
export function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
|
||||
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
|
||||
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
|
||||
}
|
||||
|
||||
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
|
||||
|
||||
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
|
||||
|
||||
export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
const withNewerRun = new Set<string>();
|
||||
const keyed = failedRuns.filter(run => run.issueId !== null);
|
||||
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
|
||||
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
|
||||
}
|
||||
const unkeyed = failedRuns.filter(run => run.issueId === null);
|
||||
if (unkeyed.length === 0) return withNewerRun;
|
||||
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
|
||||
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
|
||||
unkeyed[0]!.createdAt);
|
||||
const latestByAgent = new Map<string, Date>();
|
||||
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const rows = await db.select({
|
||||
agentId: heartbeatRuns.agentId,
|
||||
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
|
||||
gt(heartbeatRuns.createdAt, oldestCreatedAt),
|
||||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
|
||||
)).groupBy(heartbeatRuns.agentId);
|
||||
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
|
||||
}
|
||||
for (const run of unkeyed) {
|
||||
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
|
||||
}
|
||||
return withNewerRun;
|
||||
}
|
||||
|
||||
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
if (failedRuns.length === 0) return [];
|
||||
const failed = sql.join(
|
||||
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
|
||||
sql`, `,
|
||||
);
|
||||
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
|
||||
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
|
||||
const newerRunExists = (match: SQL) => sql`exists (
|
||||
select 1 from ${heartbeatRuns}
|
||||
where ${heartbeatRuns.companyId} = ${companyId}
|
||||
and ${heartbeatRuns.agentId} = failed.agent_id
|
||||
and ${heartbeatRuns.createdAt} >= failed.newer_from
|
||||
and ${match}
|
||||
)`;
|
||||
const rows = await db.execute(sql`
|
||||
select failed.id
|
||||
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
|
||||
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
|
||||
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
|
||||
`) as unknown as Array<{ id: string }>;
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
|
||||
export function requiresReviewAttention(input: {
|
||||
hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean;
|
||||
}) {
|
||||
return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled;
|
||||
}
|
||||
|
||||
export function terminalBlockerRequiresAttention<T extends { state?: string; blockingTreeLive?: boolean }>(
|
||||
attention: T | null | undefined,
|
||||
): attention is T & { state: "stalled" | "needs_attention" } {
|
||||
return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? ""));
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
|
||||
import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js";
|
||||
import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agents,
|
||||
@@ -58,7 +59,7 @@ import {
|
||||
BLOCKER_ATTENTION_MAX_NODES,
|
||||
issueService,
|
||||
} from "./issues.js";
|
||||
import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js";
|
||||
import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js";
|
||||
import { parseIssueExecutionState } from "./issue-execution-policy.js";
|
||||
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
|
||||
import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js";
|
||||
@@ -107,7 +108,6 @@ const SOURCE_RANK: Record<AttentionSourceKind, number> = {
|
||||
|
||||
const PENDING_INTERACTION_STATUSES = ["pending"] as const;
|
||||
const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const;
|
||||
const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
|
||||
const DETAIL_EXCERPT_LENGTH = 160;
|
||||
const DETAIL_IMAGE_LIMIT = 3;
|
||||
const OPEN_DECISION_DEFAULT_LIMIT = 500;
|
||||
@@ -766,37 +766,6 @@ export function interactionResolverAudience(
|
||||
};
|
||||
}
|
||||
|
||||
function collapsePendingConfirmationsToNewest<T extends {
|
||||
id: string;
|
||||
issueId: string;
|
||||
kind: string;
|
||||
createdAt: Date;
|
||||
payload: unknown;
|
||||
}>(rows: T[]) {
|
||||
// Quality gates bind independent outputs; a newer visual approval must not
|
||||
// hide a still-pending copy approval (or affect its badge count).
|
||||
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
|
||||
const newestByGroup = new Map<string, T>();
|
||||
for (const row of rows) {
|
||||
if (row.kind !== "request_confirmation" || independent(row)) continue;
|
||||
const groupKey = `${row.issueId}:${row.kind}`;
|
||||
const newest = newestByGroup.get(groupKey);
|
||||
if (
|
||||
!newest
|
||||
|| row.createdAt.getTime() > newest.createdAt.getTime()
|
||||
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
|
||||
) {
|
||||
newestByGroup.set(groupKey, row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows.filter((row) => (
|
||||
row.kind !== "request_confirmation"
|
||||
|| independent(row)
|
||||
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
|
||||
));
|
||||
}
|
||||
|
||||
function budgetObservedPercent(amountObserved: number, amountLimit: number) {
|
||||
return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0;
|
||||
}
|
||||
@@ -866,9 +835,7 @@ async function issueSummaryMap(
|
||||
.where(and(
|
||||
eq(issues.companyId, companyId),
|
||||
inArray(issues.id, ids),
|
||||
includeConversations
|
||||
? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))
|
||||
: executionIssueCondition(),
|
||||
attentionIssueCondition(includeConversations),
|
||||
readCondition,
|
||||
));
|
||||
return new Map(rows.map((row) => [row.id, {
|
||||
@@ -1108,70 +1075,6 @@ function resolveBlockingIssue(
|
||||
return null;
|
||||
}
|
||||
|
||||
function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
|
||||
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
|
||||
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
|
||||
}
|
||||
|
||||
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
|
||||
|
||||
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
|
||||
|
||||
async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
const withNewerRun = new Set<string>();
|
||||
const keyed = failedRuns.filter(run => run.issueId !== null);
|
||||
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
|
||||
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
|
||||
}
|
||||
const unkeyed = failedRuns.filter(run => run.issueId === null);
|
||||
if (unkeyed.length === 0) return withNewerRun;
|
||||
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
|
||||
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
|
||||
unkeyed[0]!.createdAt);
|
||||
const latestByAgent = new Map<string, Date>();
|
||||
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const rows = await db.select({
|
||||
agentId: heartbeatRuns.agentId,
|
||||
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
|
||||
gt(heartbeatRuns.createdAt, oldestCreatedAt),
|
||||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
|
||||
)).groupBy(heartbeatRuns.agentId);
|
||||
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
|
||||
}
|
||||
for (const run of unkeyed) {
|
||||
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
|
||||
}
|
||||
return withNewerRun;
|
||||
}
|
||||
|
||||
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
if (failedRuns.length === 0) return [];
|
||||
const failed = sql.join(
|
||||
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
|
||||
sql`, `,
|
||||
);
|
||||
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
|
||||
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
|
||||
const newerRunExists = (match: SQL) => sql`exists (
|
||||
select 1 from ${heartbeatRuns}
|
||||
where ${heartbeatRuns.companyId} = ${companyId}
|
||||
and ${heartbeatRuns.agentId} = failed.agent_id
|
||||
and ${heartbeatRuns.createdAt} >= failed.newer_from
|
||||
and ${match}
|
||||
)`;
|
||||
const rows = await db.execute(sql`
|
||||
select failed.id
|
||||
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
|
||||
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
|
||||
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
|
||||
`) as unknown as Array<{ id: string }>;
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) {
|
||||
const openDecisionLimit = Math.min(
|
||||
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
|
||||
@@ -1636,8 +1539,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
}));
|
||||
}
|
||||
const blockerAttention = issue.blockerAttention;
|
||||
if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue;
|
||||
if (blockerAttention.blockingTreeLive) continue;
|
||||
if (!terminalBlockerRequiresAttention(blockerAttention)) continue;
|
||||
const issueSummary = blockedIssueSummaries.get(issue.id) ?? null;
|
||||
const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id;
|
||||
const terminalSummary = terminalBlockerSummaries.get(terminalIssueId)
|
||||
@@ -1739,7 +1641,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null;
|
||||
const reviewAttention = reviewAttentionByIssueId.get(review.id);
|
||||
const stalled = reviewAttention?.state === "stalled";
|
||||
if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue;
|
||||
if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId),
|
||||
hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue;
|
||||
const issue = reviewIssueMap.get(review.id);
|
||||
if (!issue) continue;
|
||||
const dedupKey = `review:${review.id}`;
|
||||
@@ -1858,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
const budgetOverview = await budgetService(db).overview(companyId);
|
||||
for (const incident of budgetOverview.activeIncidents) {
|
||||
const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit);
|
||||
if (incident.thresholdType !== "hard" && observedPercent < 85) continue;
|
||||
if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue;
|
||||
const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`;
|
||||
add(createItem({
|
||||
companyId,
|
||||
@@ -1921,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
updatedAt: agents.updatedAt,
|
||||
})
|
||||
.from(agents)
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.status, "error")))
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS)))
|
||||
.orderBy(desc(agents.updatedAt), desc(agents.id));
|
||||
|
||||
for (const agent of erroredAgents) {
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyApprovalWork } from "./approval-work-notifications.js";
|
||||
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { addCents, compareCents, normalizeCents } from "@paperclipai/shared";
|
||||
import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -163,6 +166,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise<ScopeRecord> {
|
||||
let scope: ScopeRecord;
|
||||
try {
|
||||
scope = await resolveScopeRecord(db, scopeType, scopeId);
|
||||
} catch (error) {
|
||||
if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error;
|
||||
return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project",
|
||||
paused: false, pauseReason: null, budgetHeld: false };
|
||||
}
|
||||
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
|
||||
return scope;
|
||||
}
|
||||
|
||||
// Keep a large dashboard from filling the database pool with queued reads.
|
||||
async function mapBudgetReads<T, R>(rows: T[], read: (row: T) => Promise<R>): Promise<R[]> {
|
||||
const result: R[] = [];
|
||||
@@ -287,6 +303,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -300,7 +318,10 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.paused) return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
|
||||
@@ -346,6 +367,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -360,7 +383,10 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync);
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.pauseReason !== "budget") return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
|
||||
@@ -395,7 +421,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
|
||||
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
const observed = await computeObservedSpend(db, policy);
|
||||
const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed;
|
||||
const { windowStart: start, windowEnd: end } = observed;
|
||||
@@ -466,6 +492,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
windowEnd: end,
|
||||
});
|
||||
|
||||
await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" });
|
||||
if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId);
|
||||
const approval = thresholdType === "hard"
|
||||
? await db
|
||||
@@ -504,7 +531,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
return incident ? { incident, created: true } : null;
|
||||
}
|
||||
|
||||
async function resolveOpenSoftIncidents(policyId: string) {
|
||||
async function resolveOpenSoftIncidents(policyId: string, companyId: string) {
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
await db
|
||||
.update(budgetIncidents)
|
||||
.set({
|
||||
@@ -523,10 +551,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
async function resolveOpenIncidentsForPolicy(
|
||||
policyId: string,
|
||||
companyId: string,
|
||||
approvalStatus: "approved" | "rejected" | null,
|
||||
decidedByUserId: string | null,
|
||||
thresholdType?: BudgetThresholdType,
|
||||
) {
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"),
|
||||
thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined);
|
||||
const openRows = await db
|
||||
@@ -560,7 +590,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
|
||||
|
||||
return mapBudgetReads(rows, async (row) => {
|
||||
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
|
||||
const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId);
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
@@ -615,9 +645,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy;
|
||||
// A closed UTC window, inactive policy, or budget raise releases only
|
||||
// budget-owned pauses. Manual pauses remain operator decisions.
|
||||
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard");
|
||||
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard");
|
||||
if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) {
|
||||
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft");
|
||||
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft");
|
||||
}
|
||||
}
|
||||
// Agent status has its own admission gate and error code. Preserve a
|
||||
@@ -674,7 +704,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (observedBlocks(policy, observed)) {
|
||||
await resolveOpenSoftIncidents(policy.id);
|
||||
await resolveOpenSoftIncidents(policy.id, policy.companyId);
|
||||
const hardIncident = await createIncidentIfNeeded(policy, "hard", observed);
|
||||
await pauseAndCancelScopeForBudget(policy);
|
||||
if (hardIncident?.created) {
|
||||
@@ -844,11 +874,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const observed = await computeObservedSpend(db, row);
|
||||
if (!observedBlocks(row, observed)) {
|
||||
await resumeScopeFromBudget(row);
|
||||
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard");
|
||||
if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) {
|
||||
await createIncidentIfNeeded(row, "soft", observed);
|
||||
} else {
|
||||
await resolveOpenIncidentsForPolicy(row.id, null, null, "soft");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft");
|
||||
}
|
||||
} else {
|
||||
const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100);
|
||||
@@ -856,14 +886,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
await createIncidentIfNeeded(row, "soft", observed);
|
||||
}
|
||||
if (observedBlocks(row, observed)) {
|
||||
await resolveOpenSoftIncidents(row.id);
|
||||
await resolveOpenSoftIncidents(row.id, row.companyId);
|
||||
await createIncidentIfNeeded(row, "hard", observed);
|
||||
await pauseAndCancelScopeForBudget(row);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await resumeScopeFromBudget(row);
|
||||
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service");
|
||||
}
|
||||
|
||||
await recordActivity({
|
||||
@@ -925,6 +955,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
if (incident.companyId !== companyId) throw notFound("Budget incident not found");
|
||||
|
||||
const policy = await getPolicyRow(incident.policyId);
|
||||
if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) {
|
||||
throw notFound("Budget scope not found");
|
||||
}
|
||||
const scope = input.action === "keep_paused"
|
||||
? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId)
|
||||
: await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
|
||||
if (input.action === "raise_budget_and_resume") {
|
||||
const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
|
||||
const observed = await computeObservedSpend(db, policy);
|
||||
@@ -937,6 +974,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId);
|
||||
await db
|
||||
.update(budgetPolicies)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
|
||||
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
import { createFailedRetryNotices } from "./failed-retry-notices.js";
|
||||
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
// Source authorization above checked every external receipt and its owner.
|
||||
// Reset only this certified unstarted batch under the task's admission lock.
|
||||
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
|
||||
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
|
||||
const settle = async (status: string, result: Record<string, unknown>) => {
|
||||
const [current] = await db.transaction(async (tx) => {
|
||||
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
|
||||
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
|
||||
});
|
||||
await notifyChatActionWork(tx, "failed_run_retry");
|
||||
if (status === "processed") await notifyNativeResponsePresentation(tx, {
|
||||
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import { z } from "zod";
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
|
||||
)).for("update");
|
||||
if (receipt?.runId) return;
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "../../status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "../../issue-recovery-notifications.js";
|
||||
import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js";
|
||||
|
||||
import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js";
|
||||
@@ -78,10 +80,12 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm
|
||||
})(),
|
||||
);
|
||||
const effect = await db.transaction(async (tx) => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.endpoint.companyId });
|
||||
// Replace the optimistic attachment placeholder with an explicit safe
|
||||
// Paperclip-visible failure. The message link stays authoritative for
|
||||
// exact provider redelivery, while no agent wakeup is enqueued for an
|
||||
// empty turn that contains no usable bytes.
|
||||
await notifyStatusCardIssueChange(tx, input.endpoint.companyId);
|
||||
await tx
|
||||
.update(issueComments)
|
||||
.set({ body: visibleFailure, updatedAt: new Date() })
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyIssueReviewState } from "./issue-review-state-notifications.js";
|
||||
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
|
||||
import { notifyChatResponseAvailability } from "./chat-response-availability.js";
|
||||
@@ -11,6 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
|
||||
export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string };
|
||||
|
||||
export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
|
||||
if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope);
|
||||
if (scope.issueId || scope.runId) {
|
||||
await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId,
|
||||
runId: scope.issueId ? undefined : scope.runId });
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
|
||||
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
@@ -332,10 +335,13 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
|
||||
const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch);
|
||||
|
||||
if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
|
||||
await notifyRunDispatch(tx, { companyId: id });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, id);
|
||||
await notifyCompanyRetentionWork(tx, id);
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id);
|
||||
@@ -488,10 +494,13 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
|
||||
const wasAlreadyArchived = existing.status === "archived";
|
||||
|
||||
if (!wasAlreadyArchived) {
|
||||
await notifyRunDispatch(tx, { companyId: id });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, id);
|
||||
await notifyCompanyRetentionWork(tx, id);
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
|
||||
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
|
||||
@@ -71,12 +73,14 @@ export async function deleteCompany(db: Db, id: string) {
|
||||
const [existing] = await tx.select({ id: companies.id }).from(companies)
|
||||
.where(eq(companies.id, id)).for("no key update");
|
||||
if (!existing) return null;
|
||||
await notifyIssueRecovery(tx, { companyId: id });
|
||||
await notifyChatCompanyConfiguration(tx, id);
|
||||
await chatCompanyDeletion.deleteCompanyData(tx, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardEvaluation(tx, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry, id);
|
||||
await notifyToolReviewChange(tx, id);
|
||||
await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { createAgentLifecycle } from "./agent-lifecycle.js";
|
||||
import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared";
|
||||
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
@@ -6465,7 +6467,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
}
|
||||
if (relationRows.length > 0) {
|
||||
const relationCompanyId = targetCompany.id;
|
||||
await db
|
||||
await db.transaction(async tx => {
|
||||
await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId });
|
||||
await notifyIssueRecovery(tx, { companyId: relationCompanyId });
|
||||
await tx
|
||||
.insert(issueRelations)
|
||||
.values(relationRows.map((row) => ({
|
||||
companyId: relationCompanyId,
|
||||
@@ -6476,6 +6481,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
createdByUserId: actorUserId ?? null,
|
||||
})))
|
||||
.onConflictDoNothing();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -207,14 +207,20 @@ function priorityRank(priority: string | null | undefined) {
|
||||
return index >= 0 ? index : ISSUE_PRIORITIES.length;
|
||||
}
|
||||
|
||||
function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
|
||||
/** Duration used by saved search membership and its deadline consumers. */
|
||||
export function companySearchUpdatedWithinMs(value: string | undefined): number | null {
|
||||
if (!value) return null;
|
||||
const match = /^(\d+)(h|d|w|m)$/.exec(value);
|
||||
if (!match) return null;
|
||||
const amount = Number.parseInt(match[1]!, 10);
|
||||
const unit = match[2];
|
||||
const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30;
|
||||
return new Date(now.getTime() - hours * 60 * 60 * 1000);
|
||||
return hours * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
|
||||
const duration = companySearchUpdatedWithinMs(value);
|
||||
return duration === null ? null : new Date(now.getTime() - duration);
|
||||
}
|
||||
|
||||
function issueOnlyFiltersActive(query: CompanySearchQuery) {
|
||||
@@ -542,7 +548,7 @@ export function companySearchService(db: Db) {
|
||||
search: async (
|
||||
companyId: string,
|
||||
query: CompanySearchQuery,
|
||||
options?: { issueReadCondition?: SQL<boolean>; projectReadCondition?: SQL<boolean> },
|
||||
options?: { issueReadCondition?: SQL<boolean>; projectReadCondition?: SQL<boolean>; onNextMembershipChange?: (at: Date | null) => void },
|
||||
): Promise<CompanySearchResponse> => {
|
||||
const taskSearch = parseTaskSearch(query.q);
|
||||
const normalizedQuery = taskSearch.normalizedQuery;
|
||||
@@ -553,6 +559,7 @@ export function companySearchService(db: Db) {
|
||||
const limit = query.limit;
|
||||
const offset = query.offset;
|
||||
if (!hasSearchText && !issueOnlyFiltersActive(query)) {
|
||||
options?.onNextMembershipChange?.(null);
|
||||
return {
|
||||
query: query.q,
|
||||
normalizedQuery,
|
||||
@@ -721,6 +728,13 @@ export function companySearchService(db: Db) {
|
||||
}
|
||||
}
|
||||
|
||||
const membershipWindow = companySearchUpdatedWithinMs(query.updatedWithin);
|
||||
if (options?.onNextMembershipChange && membershipWindow !== null && scopeIncludesIssues(scope)) {
|
||||
// Includes rows outside the visible page: their expiry can move an
|
||||
// offset/priority-ranked result into the watched set without a write.
|
||||
branches.push(sql`SELECT 'membership-expiry' AS kind, min(m.updated_at)::text AS value,
|
||||
0::int AS count ${countTail} FROM matched m ${branchWhere([scopeCond, ...facetsAll])}`);
|
||||
}
|
||||
const resultRows = await db.execute(sql`
|
||||
${taskSearchCtes(companyId, taskSearch, scope !== "issues", and(...issueFilters), options?.issueReadCondition)}
|
||||
${sql.join(branches, sql` UNION ALL `)}
|
||||
@@ -732,6 +746,7 @@ export function companySearchService(db: Db) {
|
||||
totals: { current: 0, unfiltered: 0, omit: {} },
|
||||
};
|
||||
const issueRowsRaw: IssueSearchRow[] = [];
|
||||
let nextMembershipChange: Date | null = null;
|
||||
for (const row of resultRows) {
|
||||
if (row.kind === "result") {
|
||||
issueRowsRaw.push({
|
||||
@@ -756,6 +771,9 @@ export function companySearchService(db: Db) {
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (row.kind === "membership-expiry" && row.value && membershipWindow !== null) {
|
||||
nextMembershipChange = new Date(new Date(row.value).getTime() + membershipWindow + 1);
|
||||
}
|
||||
const count = Number(row.count ?? 0);
|
||||
if (row.kind === "type:issue") aggregates.typeCounts.issue = count;
|
||||
else if (row.kind === "type:comment") aggregates.typeCounts.comment = count;
|
||||
@@ -776,6 +794,7 @@ export function companySearchService(db: Db) {
|
||||
aggregates.totals.omit[row.kind.slice("total:omit:".length) as CompanySearchIssueFilterKey] = count;
|
||||
}
|
||||
}
|
||||
options?.onNextMembershipChange?.(nextMembershipChange);
|
||||
return { rows: await enrichIssueSnippets(issueRowsRaw), aggregates };
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { skillExecutionContent } from "./skill-execution-content.js";
|
||||
import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js";
|
||||
import { managedSkillContent, type SkillContentPublication } from "./managed-skill-content.js";
|
||||
@@ -7340,6 +7341,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
|
||||
for (const row of rows) {
|
||||
await db.transaction(async (tx) => {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ hiddenAt: now, updatedAt: now })
|
||||
|
||||
@@ -26,7 +26,7 @@ export function registerDeadlineWork(
|
||||
pausePassiveMaintenance: true,
|
||||
onSettled: hints => { requested++; task.onSettled?.(hints); },
|
||||
run: async signal => {
|
||||
if (!task.canRun()) { requested++; next = null; return; }
|
||||
if (!task.canRun()) { requested++; return; }
|
||||
if (Date.now() < earliest) { next = earliest; return; }
|
||||
if (Date.now() < coalesceUntil && (next === null || next > Date.now())) {
|
||||
next = next === null ? coalesceUntil : Math.min(next, coalesceUntil);
|
||||
@@ -39,8 +39,6 @@ export function registerDeadlineWork(
|
||||
if (signal.aborted || !task.canRun()) { next = null; return; }
|
||||
const deadline = await task.next();
|
||||
|
||||
|
||||
|
||||
if (!task.canRun()) { next = null; return; }
|
||||
earliest = deadline === null || deadline > Date.now()
|
||||
? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0)
|
||||
@@ -49,8 +47,13 @@ export function registerDeadlineWork(
|
||||
coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0);
|
||||
reconciled = generation;
|
||||
},
|
||||
hasPending: async () => requested !== reconciled,
|
||||
nextRunAt: () => next,
|
||||
// A settled producer must release its admission hold during drain. Keep
|
||||
// the unreconciled generation and an in-memory resume wake, without asking
|
||||
// the domain to probe its database while dispatch is closed.
|
||||
hasPending: async () => task.canRun() && requested !== reconciled,
|
||||
nextRunAt: () => !task.canRun() && requested !== reconciled
|
||||
? Date.now() + task.retryMs
|
||||
: next,
|
||||
});
|
||||
return { ...worker, wake: () => { requested++; worker.wake(); } };
|
||||
}
|
||||
@@ -23,7 +23,7 @@ export function decisionExpiryService(db: Db, deliverContinuation: (id: string)
|
||||
agentId: row.originAgentId, runId: row.originRunId, action: "decision.expired", entityType: "decision",
|
||||
entityId: row.id, details: { expiredReason: reason } }, publications);
|
||||
return row;
|
||||
});
|
||||
}, { companyId: decision.companyId, sourceId: decision.id });
|
||||
if (!updated) return 0;
|
||||
for (const publication of publications) publishActivity(publication);
|
||||
await deliverContinuation(updated.id);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
|
||||
import { notifyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { decisionArchiveDeliveryService, type ArchiveNotificationBatch } from "./decision-archive-delivery.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
@@ -15,7 +16,6 @@ import {
|
||||
issueRecoveryActions,
|
||||
issueThreadInteractions,
|
||||
issues,
|
||||
joinRequests,
|
||||
} from "@paperclipai/db";
|
||||
import type {
|
||||
AttentionArchiveManifestEntry,
|
||||
@@ -145,7 +145,7 @@ export function decisionRetentionService(
|
||||
const existing = new Set(rows.map(row => sourceKey(row.sourceKind, row.sourceId)));
|
||||
const missing = unique.filter(item => !existing.has(itemSourceKey(item)));
|
||||
if (missing.length) {
|
||||
for (const kind of new Set(missing.map(item => item.sourceKind))) await notifyRetentionWork(tx, kind);
|
||||
for (const item of missing) await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id });
|
||||
await tx.insert(decisionRetention).values(missing.map(item => ({
|
||||
companyId, sourceKind: item.sourceKind, sourceId: item.subject.id, sourceActivityAt: new Date(item.activityAt),
|
||||
}))).onConflictDoNothing({ target: [decisionRetention.companyId, decisionRetention.sourceKind, decisionRetention.sourceId] });
|
||||
@@ -155,7 +155,7 @@ export function decisionRetentionService(
|
||||
for (const item of unique) {
|
||||
const key = itemSourceKey(item), row = byKey.get(key), activityAt = new Date(item.activityAt);
|
||||
if (!row || row.sourceActivityAt.getTime() === activityAt.getTime()) continue;
|
||||
await notifyRetentionWork(tx, item.sourceKind);
|
||||
await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id });
|
||||
const [updated] = await tx.update(decisionRetention).set({
|
||||
sourceActivityAt: activityAt, version: sql`${decisionRetention.version} + 1`, updatedAt: new Date(),
|
||||
}).where(and(eq(decisionRetention.id, row.id), ne(decisionRetention.sourceActivityAt, activityAt))).returning();
|
||||
@@ -169,9 +169,9 @@ export function decisionRetentionService(
|
||||
return sourceKey(item.sourceKind, item.subject.id);
|
||||
}
|
||||
|
||||
async function writeRetention<T>(kind: AttentionSourceKind, write: (tx: Db) => Promise<T>) {
|
||||
async function writeRetention<T>(companyId: string, kind: AttentionSourceKind, sourceId: string, write: (tx: Db) => Promise<T>) {
|
||||
return db.transaction(async tx => {
|
||||
await notifyRetentionWork(tx, kind);
|
||||
await notifyRetentionWork(tx, kind, { companyId, sourceId });
|
||||
return write(tx as unknown as Db);
|
||||
});
|
||||
}
|
||||
@@ -185,13 +185,28 @@ export function decisionRetentionService(
|
||||
}
|
||||
|
||||
async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string }) {
|
||||
const table = input.sourceKind === "approval" ? approvals : input.sourceKind === "join_request" ? joinRequests : null;
|
||||
if (!table) return;
|
||||
const [source] = await database.select({ updatedAt: table.updatedAt }).from(table)
|
||||
.where(and(eq(table.companyId, input.companyId), eq(table.id, input.sourceId))).for("update");
|
||||
const tables: Record<AttentionSourceKind, string> = {
|
||||
approval: "approvals", join_request: "join_requests", decision: "decisions",
|
||||
issue_thread_interaction: "issue_thread_interactions", recovery_action: "issue_recovery_actions",
|
||||
failed_run: "heartbeat_runs", productivity_review: "issues", blocker_attention: "issues", review: "issues",
|
||||
budget_alert: "budget_incidents", agent_error_alert: "agents",
|
||||
};
|
||||
if (input.sourceKind === "blocker_attention") {
|
||||
const [source] = await database.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceId))).for("update");
|
||||
if (!source) throw notFound("Attention source not found");
|
||||
const activityAt = source.unblockDescriptor?.owner === "board" && isProspectiveBlockedTransition(source)
|
||||
? source.blockedTransitionAt : source.updatedAt;
|
||||
await decisionRetentionService(database).syncItems(input.companyId, [{
|
||||
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: activityAt.toISOString(),
|
||||
}]);
|
||||
return;
|
||||
}
|
||||
const activity = input.sourceKind === "failed_run" ? sql`coalesce(finished_at, updated_at, created_at)` : sql`updated_at`;
|
||||
const [source] = await database.execute(sql`select ${activity} as "activityAt" from ${sql.identifier(tables[input.sourceKind])}
|
||||
where company_id = ${input.companyId}::uuid and id = ${input.sourceId}::uuid for update`) as unknown as Array<{ activityAt: Date | string }>;
|
||||
if (!source) throw notFound("Attention source not found");
|
||||
await decisionRetentionService(database).syncItems(input.companyId, [{
|
||||
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(),
|
||||
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: new Date(source.activityAt).toISOString(),
|
||||
}]);
|
||||
}
|
||||
|
||||
@@ -206,7 +221,7 @@ export function decisionRetentionService(
|
||||
if (!(await canReadDecisionSource(db, input.authActor, input.companyId, input.sourceKind, input.sourceId))) {
|
||||
throw notFound("Attention source not found");
|
||||
}
|
||||
const updated = await writeRetention(input.sourceKind, async tx => {
|
||||
const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => {
|
||||
await materializeSourceState(tx, input);
|
||||
return tx.update(decisionRetention).set({
|
||||
keep: input.keep,
|
||||
@@ -247,7 +262,7 @@ export function decisionRetentionService(
|
||||
throw notFound("Attention source not found");
|
||||
}
|
||||
const now = new Date();
|
||||
const updated = await writeRetention(input.sourceKind, async tx => {
|
||||
const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => {
|
||||
await materializeSourceState(tx, input);
|
||||
return tx.update(decisionRetention).set(input.archived ? {
|
||||
archivedAt: now,
|
||||
@@ -324,7 +339,7 @@ export function decisionRetentionService(
|
||||
}
|
||||
const now = new Date();
|
||||
for (const entry of input.manifest) {
|
||||
await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind);
|
||||
await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind, { companyId: input.companyId, sourceId: entry.sourceId });
|
||||
const archived = await txDb.update(decisionRetention).set({
|
||||
archivedAt: now,
|
||||
archivedReason: "accepted_proposal",
|
||||
@@ -368,7 +383,7 @@ export function decisionRetentionService(
|
||||
await db.transaction(async (tx) => {
|
||||
const txDb = tx as unknown as Db;
|
||||
for (const item of candidates) {
|
||||
await notifyRetentionWork(txDb, item.sourceKind);
|
||||
await notifyRetentionWork(txDb, item.sourceKind, { companyId: input.companyId, sourceId: item.subject.id });
|
||||
const updated = await txDb.update(decisionRetention).set({
|
||||
archivedAt: now,
|
||||
archivedReason: "idle_ttl",
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export function notifyDecisionCreated(transaction: object) {
|
||||
type DecisionWorkScope = { companyId: string; sourceId?: string };
|
||||
|
||||
export async function notifyDecisionCreated(transaction: object, scope: DecisionWorkScope) {
|
||||
await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" });
|
||||
return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry);
|
||||
}
|
||||
|
||||
export async function notifyDecisionChange(transaction: object) {
|
||||
export async function notifyDecisionChange(transaction: object, scope: DecisionWorkScope) {
|
||||
await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" });
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry);
|
||||
}
|
||||
|
||||
export function writeDecisionChange<T>(db: Db, write: (transaction: Db) => Promise<T>): Promise<T> {
|
||||
export function writeDecisionChange<T>(db: Db, write: (transaction: Db) => Promise<T>, scope: DecisionWorkScope): Promise<T> {
|
||||
return db.transaction(async transaction => {
|
||||
await notifyDecisionChange(transaction);
|
||||
await notifyDecisionChange(transaction, scope);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery);
|
||||
return write(transaction as unknown as Db);
|
||||
});
|
||||
|
||||
@@ -239,7 +239,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
...(input.additionalTargetSnapshots ?? {}),
|
||||
};
|
||||
const id = randomUUID();
|
||||
await notifyDecisionCreated(dbOrTx);
|
||||
await notifyDecisionCreated(dbOrTx, { companyId: input.companyId, sourceId: id });
|
||||
await notifyDecisionTargetWork(dbOrTx, `decision:${id}`);
|
||||
const [created] = await dbOrTx.insert(decisions).values({ id, companyId: input.companyId, bundleId: input.bundleId ?? null,
|
||||
originAgentId: input.agentId, originIssueId: provenance.issueId, originRunId: input.runId, ruleKey: input.ruleKey ?? null,
|
||||
@@ -581,7 +581,8 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
return attempt;
|
||||
}
|
||||
|
||||
async function settleInvalidEffectOutcome(id: string) {
|
||||
async function settleInvalidEffectOutcome(decision: typeof decisions.$inferSelect) {
|
||||
const id = decision.id;
|
||||
const publications: ActivityPublication[] = [];
|
||||
await writeDecisionChange(db, async tx => {
|
||||
const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update");
|
||||
@@ -601,14 +602,14 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
action: "decision.recovery_failed", entityType: "decision", entityId: id,
|
||||
details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId },
|
||||
}, publications);
|
||||
});
|
||||
}, { companyId: decision.companyId, sourceId: id });
|
||||
for (const publication of publications) publishActivity(publication);
|
||||
return outcome(id);
|
||||
}
|
||||
|
||||
async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) {
|
||||
const option = decision.options.find((item) => item.id === decision.chosenOptionId);
|
||||
if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id);
|
||||
if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision);
|
||||
const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null);
|
||||
const metadata = decision.metadata as Record<string, unknown>;
|
||||
if (metadata.kind === "attention_archive_proposal") {
|
||||
@@ -618,7 +619,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
targetSnapshots: decision.targetSnapshots as Record<string, Snapshot>,
|
||||
}), decision.signedSpec)) {
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "invalid_signature", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
if (option.id === "archive") {
|
||||
@@ -629,7 +630,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
const manifestHash = manifest ? hashAttentionArchiveManifest(manifest) : null;
|
||||
if (!manifest || manifestHash !== metadata.manifestHash) {
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "manifest_mismatch", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
const originActor: AuthorizationActor = {
|
||||
@@ -651,7 +652,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "archive_proposal_failed";
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: message, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
}
|
||||
@@ -659,7 +660,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
executionStatus: "succeeded",
|
||||
updatedAt: new Date(),
|
||||
metadata: { ...metadata, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) },
|
||||
}).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
}).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
for (let index = 0; index < option.effects.length; index += 1) await executeEffect(decision, option.effects[index]!, index, userActor, decision.decidedByUserId, run?.responsibleUserId ?? null);
|
||||
@@ -667,7 +668,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
const successful = rows.filter((row) => row.status === "executed").length;
|
||||
const status = rows.every((row) => row.status === "executed") ? "succeeded" : successful ? "partial" : "failed";
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: status, updatedAt: new Date(), metadata: { ...decision.metadata,
|
||||
...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
|
||||
@@ -706,7 +707,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
decideIdempotencyKey: input.idempotencyKey ?? null,
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}),
|
||||
...(input.dismissed ? { dismissed: true, dismissReason: input.dismissReason ?? null } : {}) } })
|
||||
.where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning());
|
||||
.where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: current.id });
|
||||
if (!claimed) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
|
||||
const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed.originRunId)).then((rows) => rows[0] ?? null);
|
||||
await logActivity(db, { companyId: claimed.companyId, actorType: "system", actorId: "decision-executor", agentId: claimed.originAgentId, runId: claimed.originRunId,
|
||||
@@ -724,7 +725,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
return (await get(id))!;
|
||||
}
|
||||
const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "cancelled", updatedAt: new Date(), metadata: { ...current.metadata,
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning());
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id });
|
||||
if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
|
||||
await logActivity(db, { companyId: updated.companyId, actorType: actor.actorType, actorId: actor.actorId, runId: actor.runId, action: "decision.cancelled", entityType: "decision", entityId: id });
|
||||
await continuations.deliver(updated.id);
|
||||
@@ -738,7 +739,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
if (empty) return decide({ id, optionId: empty.id, decidedByUserId: userId, userActor, dismissed: true, dismissReason: reason });
|
||||
const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "succeeded", chosenOptionId: "dismissed", decidedByUserId: userId,
|
||||
decidedAt: new Date(), updatedAt: new Date(), metadata: { ...current.metadata, dismissed: true, dismissReason: reason ?? null,
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning());
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id });
|
||||
if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
|
||||
await logActivity(db, { companyId: updated.companyId, actorType: "system", actorId: "decision-executor", agentId: updated.originAgentId,
|
||||
runId: updated.originRunId, responsibleUserIdOverride: userId, action: "decision.dismissed", entityType: "decision", entityId: updated.id,
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export type DeferredExecutionScope = { companyId: string; requestId?: string };
|
||||
export type DeferredReceiptRecoveryOptions = {
|
||||
companyId?: string;
|
||||
requestId?: string;
|
||||
onError?: (error: unknown) => void;
|
||||
};
|
||||
|
||||
function hint(scope: DeferredExecutionScope) {
|
||||
return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
|
||||
}
|
||||
|
||||
export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
|
||||
if (scope) {
|
||||
await notifyDeferredExecutionWork(tx, scope);
|
||||
await notifyNativeStatusWakeWork(tx, scope);
|
||||
} else {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
|
||||
|
||||
export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
|
||||
condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
|
||||
return db.transaction(async tx => {
|
||||
const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
|
||||
.where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
|
||||
if (!current) return;
|
||||
const previous = current.payload?.executionWait;
|
||||
const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
|
||||
? (previous as Record<string, unknown>).reason : undefined;
|
||||
if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
|
||||
|| ${JSON.stringify(condition)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,18 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
|
||||
|
||||
/** Topics for existing durable queues; register intent before their writes. */
|
||||
export const DELIVERY_QUEUES = {
|
||||
deferredExecutionReceipts: "deferred-execution-receipts",
|
||||
taskRetryReceiptSettlement: "task-retry-receipt-settlement",
|
||||
nativeStatusWakeIntents: "native-status-wake-intents",
|
||||
orphanBlockers: "orphan-blockers",
|
||||
activeRecoveryActions: "active-recovery-actions",
|
||||
strandedAssignments: "stranded-assignments",
|
||||
resolvedDependencies: "resolved-dependencies",
|
||||
taskWatchdogs: "task-watchdogs",
|
||||
staleIssueLocks: "stale-issue-locks",
|
||||
scheduledRetryDispatch: "scheduled-retry-dispatch",
|
||||
queuedRunDispatch: "queued-run-dispatch",
|
||||
statusCardEvaluation: "status-card-evaluation",
|
||||
runEvidenceMigration: "run-evidence-migration",
|
||||
providerTraceRetention: "provider-trace-retention",
|
||||
nativeRunnerEvidenceRetention: "native-runner-evidence-retention",
|
||||
@@ -35,6 +47,7 @@ export const DELIVERY_QUEUES = {
|
||||
executionDisposition: "execution-disposition",
|
||||
environmentCleanup: "environment-cleanup",
|
||||
approvalQueueSeeding: "approval-queue-seeding",
|
||||
attentionRetention: "attention-retention",
|
||||
approvalRetention: "approval-retention",
|
||||
joinRequestRetention: "join-request-retention",
|
||||
decisionTargets: "decision-targets",
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, asc, desc, eq, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -238,6 +240,7 @@ export function documentService(db: Db) {
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
||||
try {
|
||||
return await db.transaction(async (tx) => {
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
const now = new Date();
|
||||
const existing = await tx
|
||||
.select({
|
||||
@@ -275,6 +278,7 @@ export function documentService(db: Db) {
|
||||
.where(eq(issueDocuments.issueId, issue.id));
|
||||
const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key));
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const [document] = await tx
|
||||
.insert(documents)
|
||||
.values({
|
||||
@@ -376,6 +380,7 @@ export function documentService(db: Db) {
|
||||
});
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const nextRevisionNumber = existing.latestRevisionNumber + 1;
|
||||
const [revision] = await tx
|
||||
.insert(documentRevisions)
|
||||
@@ -438,6 +443,7 @@ export function documentService(db: Db) {
|
||||
throw conflict("Document does not exist yet", { key });
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const [document] = await tx
|
||||
.insert(documents)
|
||||
.values({
|
||||
@@ -592,6 +598,10 @@ export function documentService(db: Db) {
|
||||
});
|
||||
}
|
||||
await db.transaction(async (tx) => {
|
||||
for (const companyId of new Set(rows.map(row => row.companyId))) {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
}
|
||||
await insertRowsInChunks(tx, documents, documentRows);
|
||||
await insertRowsInChunks(tx, documentRevisions, revisionRows);
|
||||
await insertRowsInChunks(tx, issueDocuments, linkRows);
|
||||
@@ -644,6 +654,7 @@ export function documentService(db: Db) {
|
||||
});
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
const now = new Date();
|
||||
const nextRevisionNumber = existing.latestRevisionNumber + 1;
|
||||
const [restoredRevision] = await tx
|
||||
@@ -662,6 +673,7 @@ export function documentService(db: Db) {
|
||||
})
|
||||
.returning();
|
||||
|
||||
await notifyStatusCardIssueChange(tx, existing.companyId);
|
||||
await tx
|
||||
.update(documents)
|
||||
.set({
|
||||
@@ -722,6 +734,7 @@ export function documentService(db: Db) {
|
||||
};
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
const now = new Date();
|
||||
await tx
|
||||
.update(documents)
|
||||
@@ -769,6 +782,7 @@ export function documentService(db: Db) {
|
||||
};
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
const now = new Date();
|
||||
await tx
|
||||
.update(documents)
|
||||
@@ -817,7 +831,9 @@ export function documentService(db: Db) {
|
||||
});
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id));
|
||||
await notifyStatusCardIssueChange(tx, existing.companyId);
|
||||
await tx.delete(documents).where(eq(documents.id, existing.id));
|
||||
|
||||
return {
|
||||
|
||||
@@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s
|
||||
import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
|
||||
import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js";
|
||||
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js";
|
||||
import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
|
||||
@@ -4021,7 +4022,12 @@ export function environmentRuntimeService(
|
||||
const lease = await environmentsSvc.getLeaseById(leaseId);
|
||||
if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined;
|
||||
const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
|
||||
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease });
|
||||
// Legacy Cloud leases can lack a credential version that permits a
|
||||
// provider-local observation. Cloud's manager verifies the historical
|
||||
// organization and signs only exact absence; no replacement key or
|
||||
// cleanup label is treated as proof. Explicit provider answers win.
|
||||
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease })
|
||||
?? await inspectCloudSandboxCleanupAbsence(lease);
|
||||
},
|
||||
|
||||
async retryPendingSandboxTeardown(input: {
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js";
|
||||
|
||||
@@ -6,6 +10,12 @@ export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?:
|
||||
|
||||
export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) {
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
if (scope) {
|
||||
await notifyAttentionRetentionWork(transaction, scope);
|
||||
await notifyStatusCardIssueChange(transaction, scope.companyId);
|
||||
await notifyRunDispatch(transaction, scope);
|
||||
await notifyDeferredExecutionEligibility(transaction, scope);
|
||||
}
|
||||
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
|
||||
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
|
||||
await notifyDeliveryWork(transaction, queue);
|
||||
@@ -20,6 +30,10 @@ export function writeExecutionControl<T>(
|
||||
): Promise<T> {
|
||||
return db.transaction(async transaction => {
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
if (scope) await notifyAttentionRetentionWork(transaction, scope);
|
||||
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
|
||||
await notifyRunDispatch(transaction, scope);
|
||||
if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
|
||||
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
|
||||
return write(transaction as unknown as Db);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js";
|
||||
import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js";
|
||||
import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
|
||||
@@ -1590,6 +1591,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
await validateSelection(input, tx);
|
||||
const intent: TaskWorkspaceIntent = { version: 1, request: { key: input.requestKey, expectedBindingRevision: input.expectedBindingRevision }, selection: input.selection, source: input.source ?? "explicit" };
|
||||
const nextRevision = task.workspaceBindingRevision + 1;
|
||||
await notifyStatusCardIssueChange(tx, task.companyId);
|
||||
await tx.update(issues).set({ workspaceBindingRevision: nextRevision,
|
||||
workspacePendingSelection: { version: 1, requestKey: input.requestKey,
|
||||
expectedBindingRevision: nextRevision, intent }, updatedAt: new Date() }).where(eq(issues.id, task.id));
|
||||
@@ -1633,6 +1635,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
}
|
||||
}
|
||||
await notifyStatusCardIssueChange(tx, task.companyId);
|
||||
await tx.update(issues).set({ ...patch, assigneeAdapterOverrides, workspaceSelection: pending.intent, workspacePendingSelection: null,
|
||||
workspaceBindingRevision: task.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, task.id));
|
||||
return true;
|
||||
@@ -1651,6 +1654,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
const pendingAfterFirstBinding = !task.executionWorkspaceId && pending?.expectedBindingRevision === task.workspaceBindingRevision
|
||||
? { ...pending, expectedBindingRevision: nextRevision }
|
||||
: pending;
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const [bound] = await tx.update(issues).set({ ...patch, executionWorkspaceId: workspaceId,
|
||||
workspaceBindingRevision: nextRevision, workspacePendingSelection: pendingAfterFirstBinding, updatedAt: new Date() })
|
||||
.where(eq(issues.id, issueId)).returning();
|
||||
@@ -2526,13 +2530,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
forceWorktreeRemoval: false,
|
||||
});
|
||||
if (cleanup.cleaned && workspace.mode === "shared_workspace") {
|
||||
await db
|
||||
await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, workspace.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ executionWorkspaceId: null, updatedAt: now() })
|
||||
.where(and(
|
||||
eq(issues.companyId, workspace.companyId),
|
||||
eq(issues.executionWorkspaceId, workspace.id),
|
||||
));
|
||||
});
|
||||
}
|
||||
const cleanupReason = [ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON, ...cleanup.warnings].join(" | ");
|
||||
if (!cleanup.cleaned || cleanup.warnings.length > 0) {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js";
|
||||
|
||||
export function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
|
||||
export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
|
||||
await notifyIssueRecovery(tx, scope);
|
||||
return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement,
|
||||
scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*");
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm";
|
||||
import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { authorizeExternalBotOperator } from "../external-bot-authority.js";
|
||||
@@ -74,6 +75,7 @@ export async function inspectGrokWorkspaceObligations(db: Db, companyId: string,
|
||||
return { ...rest, externalWorkspaceInspection: inspection };
|
||||
};
|
||||
await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id));
|
||||
if (actions.length) await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "recovery_action" });
|
||||
for (const action of actions) {
|
||||
await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(),
|
||||
nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
|
||||
@@ -9,6 +10,8 @@ import type { Db } from "@paperclipai/db";
|
||||
import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db";
|
||||
import { nativeSha256 } from "./native-runtime/canonical.js";
|
||||
|
||||
export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted";
|
||||
|
||||
export interface AppendHeartbeatRunEventInput {
|
||||
companyId: string;
|
||||
runId: string;
|
||||
@@ -74,10 +77,11 @@ export async function appendHeartbeatRunEvent(
|
||||
db: Db,
|
||||
input: AppendHeartbeatRunEventInput,
|
||||
): Promise<AppendHeartbeatRunEventResult> {
|
||||
const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX));
|
||||
const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null;
|
||||
const projectsProgress = isSafeNativeChatProgressEventType(input.eventType);
|
||||
const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent);
|
||||
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment) {
|
||||
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) {
|
||||
const allocated = db.$with("allocated_run_event", {}).as(sql`
|
||||
update ${heartbeatRuns}
|
||||
set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1,
|
||||
@@ -129,7 +133,7 @@ export async function appendHeartbeatRunEvent(
|
||||
eq(heartbeatRunEvents.runId, input.runId),
|
||||
eq(heartbeatRunEvents.agentId, input.agentId),
|
||||
eq(heartbeatRunEvents.eventType, "lifecycle"),
|
||||
sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`,
|
||||
sql`${heartbeatRunEvents.message} like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`}`,
|
||||
sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`,
|
||||
))
|
||||
.limit(1)
|
||||
@@ -187,6 +191,7 @@ export async function appendHeartbeatRunEvent(
|
||||
if (projectsResponse && run.runtimeMode === "native") {
|
||||
await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId });
|
||||
}
|
||||
if (projectsRetention) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "failed_run" });
|
||||
const seq = await allocateHeartbeatRunEventSeq(
|
||||
tx as unknown as Db,
|
||||
input.runId,
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
|
||||
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
|
||||
import { registerIssueRecoveryWork } from "./recovery/work.js";
|
||||
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
|
||||
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
|
||||
import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
|
||||
import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
|
||||
@@ -999,6 +1003,7 @@ export function heartbeatService(
|
||||
finalizeAiConnectionBusyDeferral,
|
||||
finalizeWorkspaceBusyDeferral,
|
||||
promoteDueScheduledRetries,
|
||||
nextScheduledRetryAt,
|
||||
retryScheduledRetryNow,
|
||||
scheduleBoundedRetry,
|
||||
} = createHeartbeatRetries(db, {
|
||||
@@ -1333,8 +1338,8 @@ export function heartbeatService(
|
||||
return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args);
|
||||
}
|
||||
|
||||
function resumeQueuedRunsAfterDrain() {
|
||||
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain();
|
||||
function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
|
||||
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
|
||||
activeWakeupPromises.add(promise);
|
||||
void promise.then(
|
||||
() => { activeWakeupPromises.delete(promise); },
|
||||
@@ -1609,15 +1614,15 @@ export function heartbeatService(
|
||||
void reportRunFailure(db, updated, failureReport);
|
||||
}
|
||||
|
||||
async function reconcileStrandedAssignedIssues(opts?: { skipExternalConversationSettlement?: boolean }) {
|
||||
async function reconcileStrandedAssignedIssues(opts?: Parameters<typeof recovery.reconcileStrandedAssignedIssues>[0]) {
|
||||
return recovery.reconcileStrandedAssignedIssues({
|
||||
skipExternalConversationSettlement: opts?.skipExternalConversationSettlement,
|
||||
...opts,
|
||||
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
|
||||
});
|
||||
}
|
||||
|
||||
async function sweepStaleIssueLocks() {
|
||||
return recovery.sweepStaleIssueLocks();
|
||||
async function sweepStaleIssueLocks(opts?: Parameters<typeof recovery.sweepStaleIssueLocks>[0]) {
|
||||
return recovery.sweepStaleIssueLocks(opts);
|
||||
}
|
||||
|
||||
function issueIdFromRunContext(contextSnapshot: unknown) {
|
||||
@@ -1647,10 +1652,7 @@ export function heartbeatService(
|
||||
});
|
||||
}
|
||||
|
||||
async function reconcileTaskWatchdogs(opts?: {
|
||||
companyId?: string | null;
|
||||
runId?: string | null;
|
||||
}) {
|
||||
async function reconcileTaskWatchdogs(opts?: Parameters<typeof taskWatchdogs.reconcileTaskWatchdogs>[0]) {
|
||||
return taskWatchdogs.reconcileTaskWatchdogs({
|
||||
...opts,
|
||||
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
|
||||
@@ -1675,10 +1677,7 @@ export function heartbeatService(
|
||||
return recovery.buildRunOutputSilence(run, now);
|
||||
}
|
||||
|
||||
async function reconcileResolvedDependencyWakes(opts?: {
|
||||
runId?: string | null;
|
||||
companyId?: string | null;
|
||||
}) {
|
||||
async function reconcileResolvedDependencyWakes(opts?: Parameters<typeof recovery.reconcileResolvedDependencyWakeBackstop>[0]) {
|
||||
return recovery.reconcileResolvedDependencyWakeBackstop(opts);
|
||||
}
|
||||
|
||||
@@ -2007,8 +2006,11 @@ export function heartbeatService(
|
||||
const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
|
||||
const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner
|
||||
&& typeof run.contextSnapshot?.conversationSessionGeneration === "number";
|
||||
if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId))
|
||||
&& !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) {
|
||||
const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn
|
||||
? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId })
|
||||
.from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null)
|
||||
: null;
|
||||
if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) {
|
||||
await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" });
|
||||
await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() });
|
||||
await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true });
|
||||
@@ -2204,44 +2206,6 @@ export function heartbeatService(
|
||||
context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true;
|
||||
}
|
||||
const wakeCommentId = deriveCommentId(context, null);
|
||||
const wakeCommentContext =
|
||||
issueContext && wakeCommentId
|
||||
? await db
|
||||
.select({
|
||||
id: issueComments.id,
|
||||
body: issueComments.body,
|
||||
authorType: issueComments.authorType,
|
||||
authorAgentId: issueComments.authorAgentId,
|
||||
authorUserId: issueComments.authorUserId,
|
||||
presentation: issueComments.presentation,
|
||||
metadata: issueComments.metadata,
|
||||
deletedAt: issueComments.deletedAt,
|
||||
deletedByType: issueComments.deletedByType,
|
||||
deletedByAgentId: issueComments.deletedByAgentId,
|
||||
deletedByUserId: issueComments.deletedByUserId,
|
||||
deletedByRunId: issueComments.deletedByRunId,
|
||||
sourceTrust: issueComments.sourceTrust,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.id, wakeCommentId),
|
||||
eq(issueComments.issueId, issueContext.id),
|
||||
eq(issueComments.companyId, agent.companyId),
|
||||
),
|
||||
)
|
||||
.then((rows) => {
|
||||
const row = rows[0] ?? null;
|
||||
return row?.deletedAt
|
||||
? {
|
||||
...row,
|
||||
body: "",
|
||||
presentation: null,
|
||||
metadata: null,
|
||||
}
|
||||
: row;
|
||||
})
|
||||
: null;
|
||||
let issueAssigneeOverrides =
|
||||
issueContext && issueContext.assigneeAgentId === agent.id
|
||||
? parseIssueAssigneeAdapterOverrides(
|
||||
@@ -2610,7 +2574,7 @@ export function heartbeatService(
|
||||
continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest,
|
||||
} = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, {
|
||||
run, agent,
|
||||
task: { issueRef, issueContext, wakeCommentContext, context, taskSession },
|
||||
task: { issueRef, issueContext, context, taskSession },
|
||||
policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) },
|
||||
services: { issuesSvc, getPinnedSkillTestContext },
|
||||
}));
|
||||
@@ -4246,6 +4210,30 @@ export function heartbeatService(
|
||||
computeTaskDrain,
|
||||
applyTaskDrain,
|
||||
|
||||
registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
|
||||
registerDeferredReceiptWork(db, coordinator, {
|
||||
resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
|
||||
settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
|
||||
dispatchNative: dispatchPendingNativeStatusWakeups,
|
||||
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
|
||||
}, () => canRun() && readTaskDrain(new Date()) === null),
|
||||
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
|
||||
registerIssueRecoveryWork(db, coordinator, {
|
||||
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
|
||||
orphanBlockers: recovery.reconcileUnassignedBlockingIssues,
|
||||
recoveryActions: recovery.reconcileActiveRecoveryActions,
|
||||
dependencies: reconcileResolvedDependencyWakes,
|
||||
locks: sweepStaleIssueLocks,
|
||||
watchdogs: reconcileTaskWatchdogs,
|
||||
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
|
||||
}, () => canRun() && readTaskDrain(new Date()) === null),
|
||||
registerRunDispatchWork: (coordinator: Parameters<typeof registerRunDispatchWork>[0], canRun: () => boolean) =>
|
||||
registerRunDispatchWork(coordinator, {
|
||||
promoteDueScheduledRetries,
|
||||
nextScheduledRetryAt,
|
||||
resumeQueuedRunsAfterDrain,
|
||||
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
|
||||
}, () => canRun() && readTaskDrain(new Date()) === null),
|
||||
promoteDueScheduledRetries,
|
||||
retryScheduledRetryNow,
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, asc, desc, eq, lte, sql } from "drizzle-orm";
|
||||
@@ -91,6 +92,7 @@ export function createHeartbeatAgentStatus(db: Db) {
|
||||
)).limit(1);
|
||||
const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure);
|
||||
const now = new Date();
|
||||
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
const [updated] = await tx.update(agents).set({
|
||||
status: nextStatus,
|
||||
@@ -140,6 +142,7 @@ export function createHeartbeatAgentStatus(db: Db) {
|
||||
eq(heartbeatRuns.agentId, candidate.id)))
|
||||
.orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1);
|
||||
const nextStatus = settledStatus(lastRun?.status);
|
||||
await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId });
|
||||
const [agent] = await tx.update(agents).set({ status: nextStatus,
|
||||
errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now,
|
||||
}).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId),
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
|
||||
type Entry = { id: string; companyId: string; dueAt: number };
|
||||
type NativeIntentOptions = DeferredExecutionScope & {
|
||||
onDeferredDeadline(at: number): void;
|
||||
onError(error: unknown): void;
|
||||
};
|
||||
|
||||
export function registerDeferredReceiptWork(
|
||||
db: Db,
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
owner: {
|
||||
resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
|
||||
settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
|
||||
dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
|
||||
schedulingSuppressed(): Promise<boolean>;
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
function register(kind: "deferred" | "task_chat" | "native") {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
|
||||
: kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
|
||||
return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
|
||||
queue,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
|
||||
load: async scope => {
|
||||
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
|
||||
status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
|
||||
claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
|
||||
delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
|
||||
or ${agentWakeupRequests.payload}->'executionWait' is not null`,
|
||||
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
|
||||
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
}).from(agentWakeupRequests).where(and(
|
||||
scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
|
||||
kind === "native" ? and(
|
||||
eq(agentWakeupRequests.requestedByActorType, "system"),
|
||||
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
|
||||
inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
|
||||
) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
|
||||
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
|
||||
eq(agentWakeupRequests.reason, "task_retry"),
|
||||
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
|
||||
sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
|
||||
sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
|
||||
and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
|
||||
sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
|
||||
sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
)),
|
||||
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
|
||||
const complete = rows.length < 100;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
|
||||
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
|
||||
: Date.now(),
|
||||
})), complete };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!enabled) return null;
|
||||
if (kind !== "task_chat") {
|
||||
const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
|
||||
updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
|
||||
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
|
||||
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
}).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
|
||||
eq(agentWakeupRequests.id, entry.id)));
|
||||
if (!current) return null;
|
||||
const due = kind === "native" && current.status === "claimed" && current.claimedAt
|
||||
? current.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
|
||||
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
|
||||
? current.updatedAt.getTime() + 30_000 : null;
|
||||
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
|
||||
}
|
||||
const errors: unknown[] = [];
|
||||
let next: number | null = null;
|
||||
const scope = { companyId: entry.companyId, requestId: entry.id };
|
||||
if (kind === "native") await owner.dispatchNative({ ...scope,
|
||||
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
|
||||
onError: error => { errors.push(error); },
|
||||
});
|
||||
else if (kind === "task_chat") await owner.settleTaskChat(scope);
|
||||
else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
|
||||
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
|
||||
return next === null ? null : { ...entry, dueAt: next };
|
||||
},
|
||||
}, canRun);
|
||||
}
|
||||
const workers = [register("deferred"), register("task_chat"), register("native")];
|
||||
return { ready: Promise.all(workers.map(worker => worker.ready)),
|
||||
wake() { for (const worker of workers) worker.invalidate(); },
|
||||
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { RunDispatchScope } from "../run-dispatch-notifications.js";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { registerDeadlineWork } from "../deadline-work.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
|
||||
export function registerRunDispatchWork(
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
owner: {
|
||||
promoteDueScheduledRetries(): Promise<unknown>;
|
||||
nextScheduledRetryAt(): Promise<number | null>;
|
||||
resumeQueuedRunsAfterDrain(scope?: RunDispatchScope): Promise<unknown>;
|
||||
schedulingSuppressed(): Promise<boolean>;
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
let suppressed = false;
|
||||
const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, {
|
||||
retryMs: 30_000,
|
||||
continuationMs: 1_000,
|
||||
notificationCoalesceMs: 1_000,
|
||||
canRun,
|
||||
run: async signal => {
|
||||
suppressed = await owner.schedulingSuppressed();
|
||||
if (!signal.aborted && !suppressed && canRun()) await owner.promoteDueScheduledRetries();
|
||||
},
|
||||
next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(),
|
||||
});
|
||||
const scopes = new Set<string>(["*"]);
|
||||
const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, {
|
||||
retryMs: 30_000,
|
||||
notificationCoalesceMs: 1_000,
|
||||
canRun,
|
||||
onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); },
|
||||
run: async signal => {
|
||||
const requested = scopes.has("*") ? ["*"] : [...scopes];
|
||||
if (requested[0] === "*") scopes.clear();
|
||||
else for (const hint of requested) scopes.delete(hint);
|
||||
for (let index = 0; index < requested.length; index++) {
|
||||
if (signal.aborted || !canRun()) {
|
||||
for (const hint of requested.slice(index)) scopes.add(hint);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await owner.resumeQueuedRunsAfterDrain(requested[index] === "*" ? undefined : JSON.parse(requested[index]) as RunDispatchScope);
|
||||
} catch (error) {
|
||||
for (const hint of requested.slice(index)) scopes.add(hint);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
},
|
||||
next: async () => null,
|
||||
});
|
||||
return {
|
||||
ready: Promise.all([retries.ready, queue.ready]),
|
||||
wake() { scopes.add("*"); retries.wake(); queue.wake(); },
|
||||
async stop() { await Promise.all([retries.stop(), queue.stop()]); },
|
||||
};
|
||||
}
|
||||
@@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
|
||||
const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId,
|
||||
runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId };
|
||||
const runScopedMentionedSkillKeys =
|
||||
preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
|
||||
const [mentionedSkills, githubIdentity] = await Promise.allSettled([
|
||||
preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
|
||||
db,
|
||||
companyId: agent.companyId,
|
||||
issueId,
|
||||
commentIds: extractWakeCommentIds(context),
|
||||
actor: skillActor,
|
||||
}));
|
||||
})),
|
||||
time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, {
|
||||
agentId: agent.id, responsibleUserId, allowStandingDelegation: false,
|
||||
})),
|
||||
]);
|
||||
if (mentionedSkills.status === "rejected") throw mentionedSkills.reason;
|
||||
if (githubIdentity.status === "rejected") throw githubIdentity.reason;
|
||||
const runScopedMentionedSkillKeys = mentionedSkills.value;
|
||||
const runScopedSkillKeys =
|
||||
!pinnedSkillTest && acceptedPlanContinuationWake &&
|
||||
!acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation
|
||||
? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY]
|
||||
: runScopedMentionedSkillKeys;
|
||||
const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(
|
||||
db,
|
||||
agent.companyId,
|
||||
{
|
||||
agentId: agent.id,
|
||||
responsibleUserId,
|
||||
allowStandingDelegation: false,
|
||||
},
|
||||
));
|
||||
const githubSelection = githubIdentity.value;
|
||||
const useHostGitHub =
|
||||
!githubSelection.configured &&
|
||||
trustPreset.kind === "standard" &&
|
||||
@@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
);
|
||||
const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = [];
|
||||
const skillsPrepareStartedAtMs = Date.now();
|
||||
let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs;
|
||||
let skillDeliveryConfig = effectiveResolvedConfig;
|
||||
const prepareSkills = async () => {
|
||||
|
||||
@@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills);
|
||||
const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills))
|
||||
.finally(() => { skillsPrepareEndedAtMs = Date.now(); });
|
||||
const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
|
||||
const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([
|
||||
skillsPreparation,
|
||||
assignmentsPreparation,
|
||||
assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))),
|
||||
]);
|
||||
if (preparedSkills.status === "rejected") throw preparedSkills.reason;
|
||||
if (preparedAssignments.status === "rejected") throw preparedAssignments.reason;
|
||||
if (preparedInstructions.status === "rejected") throw preparedInstructions.reason;
|
||||
const runtimeSkillEntries = preparedSkills.value;
|
||||
nativeRunnerPreparationSpans.push({
|
||||
name: "skills.prepare",
|
||||
parentName: "task.prepare",
|
||||
startedAtMs: skillsPrepareStartedAtMs,
|
||||
endedAtMs: Date.now(),
|
||||
endedAtMs: skillsPrepareEndedAtMs,
|
||||
});
|
||||
const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
|
||||
const connectorAssignments = preparedAssignments.value;
|
||||
const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments));
|
||||
// Both CLI adapters and native context materialization use the same resolved set.
|
||||
runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills);
|
||||
const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType));
|
||||
// Always replace this runtime-only field; caller wake data cannot supply skills.
|
||||
context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
|
||||
let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({
|
||||
resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }),
|
||||
let runtimeConfig = await prepareConnectionInstructionDelivery({
|
||||
resolve: async () => preparedInstructions.value,
|
||||
context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner",
|
||||
}));
|
||||
});
|
||||
const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined)
|
||||
? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id))
|
||||
: undefined;
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
|
||||
@@ -222,7 +227,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
|
||||
publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
|
||||
cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
|
||||
cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
|
||||
resumeExecutionWaitComments: () => Promise<unknown>;
|
||||
resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
|
||||
resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
|
||||
resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
|
||||
formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
|
||||
@@ -1268,6 +1273,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}
|
||||
async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) {
|
||||
if (!claimedRun || !issueId || !ownsIssue) return;
|
||||
await notifyStatusCardIssueChange(tx, run.companyId);
|
||||
await tx.update(issues).set({
|
||||
executionRunId: claimedRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(agent.name),
|
||||
@@ -1851,13 +1857,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
return cancelled;
|
||||
}
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
await resumeExecutionWaitComments();
|
||||
await resumeExecutionWaitComments(scope);
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
|
||||
.from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
|
||||
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
eq(companies.status, "active"),
|
||||
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
|
||||
lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
|
||||
@@ -1868,6 +1876,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
));
|
||||
await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
|
||||
});
|
||||
}
|
||||
@@ -1881,7 +1890,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
|
||||
sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
isNull(issues.executionRunId),
|
||||
or(and(
|
||||
sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
|
||||
@@ -1920,7 +1931,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
|
||||
if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
|
||||
await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
|
||||
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
|
||||
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
|
||||
});
|
||||
continue;
|
||||
@@ -1932,6 +1944,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
latest.errorCode === "execution_reconciliation_required";
|
||||
if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
|
||||
await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
|
||||
});
|
||||
}
|
||||
@@ -1939,6 +1952,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
|
||||
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
|
||||
}).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
|
||||
});
|
||||
}
|
||||
@@ -1956,6 +1970,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
))
|
||||
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
|
||||
.where(and(
|
||||
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
eq(heartbeatRuns.status, "cancelled"),
|
||||
eq(heartbeatRuns.runtimeMode, "legacy"),
|
||||
@@ -1964,16 +1980,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
));
|
||||
for (const run of interruptedQueues) {
|
||||
await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
|
||||
scope.onError?.(err);
|
||||
logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
await resumeDeferredExecutionReceipts();
|
||||
await resumeQueuedRunsAfterDrain();
|
||||
}
|
||||
|
||||
// Release admission promptly without first scanning unrelated recovery work.
|
||||
// Every run still passes the ordinary per-agent claim and dispatch gates.
|
||||
async function resumeQueuedRunsAfterDrain() {
|
||||
async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const queuedRuns = await db
|
||||
@@ -1983,6 +2004,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.status, "queued"),
|
||||
scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
|
||||
eq(companies.status, "active"),
|
||||
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
|
||||
),
|
||||
@@ -2505,17 +2527,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// its one durable receipt ID on a skipped wake.
|
||||
if (caseRequest) return { created: false };
|
||||
if (executionWaitRequestId) {
|
||||
await db.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
|
||||
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
|
||||
? "This task is paused. Resume it to send your saved message."
|
||||
: "Waiting for task execution to be enabled. Your message is saved."),
|
||||
})}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
|
||||
eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
|
||||
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
|
||||
? "This task is paused. Resume it to send your saved message."
|
||||
: "Waiting for task execution to be enabled. Your message is saved."),
|
||||
});
|
||||
return { created: false };
|
||||
}
|
||||
const request = {
|
||||
@@ -3403,15 +3419,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
|
||||
const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
|
||||
if (executionWaitRequestId) {
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(agentWakeupRequests.id, executionWaitRequestId));
|
||||
await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
|
||||
requestId: executionWaitRequestId }, condition, { replace: true });
|
||||
return { kind: "deferred" as const, admitted: true };
|
||||
}
|
||||
if (durableRequest || wakeCommentId ||
|
||||
hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
|
||||
readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
|
||||
await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
...durableReceiptFields,
|
||||
companyId: agent.companyId, agentId, source, triggerDetail, reason,
|
||||
@@ -3759,6 +3774,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
.from(agents)
|
||||
.where(eq(agents.id, legacyRun.agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
@@ -4030,6 +4046,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}));
|
||||
}
|
||||
if (followUp) {
|
||||
if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
|
||||
const assignmentId = followUp.assignmentId;
|
||||
const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
|
||||
wake.payload?.[followUpPayloadKey] === assignmentId &&
|
||||
@@ -4634,6 +4651,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}
|
||||
|
||||
const queueOutcome = await db.transaction(async (tx) => {
|
||||
await notifyRunDispatch(tx, { companyId: agent.companyId });
|
||||
await tx.execute(
|
||||
sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`,
|
||||
);
|
||||
@@ -4767,6 +4785,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
companyId?: string;
|
||||
limit?: number;
|
||||
staleClaimMs?: number;
|
||||
requestId?: string;
|
||||
onDeferredDeadline?: (at: number) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
} = {},
|
||||
) {
|
||||
const now = new Date();
|
||||
@@ -4779,6 +4800,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
input.companyId
|
||||
? eq(agentWakeupRequests.companyId, input.companyId)
|
||||
: undefined,
|
||||
input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
|
||||
eq(agentWakeupRequests.requestedByActorType, "system"),
|
||||
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
|
||||
inArray(agentWakeupRequests.status, ["queued", "claimed"]),
|
||||
@@ -4855,6 +4877,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
candidate.claimedAt &&
|
||||
now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
|
||||
) {
|
||||
input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
|
||||
deferred += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -5029,6 +5052,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (wakeRun) dispatched += 1;
|
||||
else deferred += 1;
|
||||
} catch (error) {
|
||||
input.onError?.(error);
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
@@ -5072,6 +5096,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
withChatControlRecoveryGate,
|
||||
trackWakeup,
|
||||
resumeQueuedRuns,
|
||||
resumeDeferredExecutionReceipts,
|
||||
resumeQueuedRunsAfterDrain,
|
||||
parseHeartbeatPolicy,
|
||||
nextTimerHeartbeatAt,
|
||||
|
||||
@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
|
||||
let settledProcessesDiscovered = false;
|
||||
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
|
||||
if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
|
||||
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
|
||||
const now = new Date();
|
||||
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
});
|
||||
if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
|
||||
await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
|
||||
await dispatchPendingNativeStatusWakeups().catch((error) => {
|
||||
if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
|
||||
logger.warn(
|
||||
{ err: error },
|
||||
"failed to dispatch persisted native status wake intents before orphan reaping",
|
||||
|
||||
@@ -35,6 +35,7 @@ function callbacks() {
|
||||
getWorktreeExecutionCutoff: vi.fn(async (): Promise<Date | null> => null),
|
||||
applyRunDispatchPostCommitEffects: vi.fn<(effects: PostCommitEffect[]) => void>(),
|
||||
runDispatch: {
|
||||
nextScheduledRetryAt: vi.fn(async () => null),
|
||||
evaluateScheduledRetryGate: vi.fn<HeartbeatRetryDependencies["runDispatch"]["evaluateScheduledRetryGate"]>(async () => ({ allowed: true })),
|
||||
promoteDueScheduledRetries: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteDueScheduledRetries"]>(async () => ({ promoted: 0, runIds: [], postCommitEffects: [] })),
|
||||
promoteScheduledRetry: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteScheduledRetry"]>(async () => ({ outcome: "not_promoted" })),
|
||||
@@ -183,7 +184,9 @@ describe("heartbeat retry module callbacks", () => {
|
||||
});
|
||||
});
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HEARTBEAT_RETRIES_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
|
||||
const describePostgres = support.supported ? describe : describe.skip;
|
||||
if (!support.supported) console.warn(`Skipping retry PostgreSQL tests: ${support.reason ?? "unsupported host"}`);
|
||||
|
||||
@@ -193,11 +196,11 @@ describePostgres("heartbeat retry module database wiring", () => {
|
||||
const now = new Date("2026-10-08T12:00:00Z");
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
|
||||
db = createDb(database.connectionString);
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
|
||||
db = createDb(external ?? database.connectionString, { maxConnections: 6 });
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(issues);
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js";
|
||||
import { infrastructureRetryDelayMs } from "../system-recovery-state.js";
|
||||
import {
|
||||
@@ -77,7 +80,7 @@ type PlanApprovalResumeFailure = {
|
||||
export interface HeartbeatRetryDependencies {
|
||||
hasCaseHandoffForRun: (run: Pick<HeartbeatRun, "id" | "companyId">) => Promise<boolean>;
|
||||
runDispatch: Pick<ReturnType<typeof createRunDispatch>,
|
||||
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">;
|
||||
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">;
|
||||
getAgent: (agentId: string) => Promise<typeof agents.$inferSelect | null | undefined>;
|
||||
getRun: (
|
||||
runId: string,
|
||||
@@ -997,6 +1000,8 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
retryContextSnapshot.previousRunId = continuation.previousRunId;
|
||||
}
|
||||
|
||||
await notifyAttentionRetentionWork(tx, { companyId: run.companyId });
|
||||
await notifyRunDispatch(tx, { companyId: run.companyId });
|
||||
const wakeupRequest = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
@@ -1178,6 +1183,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
}
|
||||
|
||||
if (issueId) {
|
||||
await notifyStatusCardIssueChange(tx, run.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
@@ -1643,6 +1649,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
});
|
||||
}
|
||||
|
||||
async function nextScheduledRetryAt() {
|
||||
return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() });
|
||||
}
|
||||
|
||||
async function promoteDueScheduledRetries(now = new Date()) {
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const result = await runDispatch.promoteDueScheduledRetries({
|
||||
@@ -1750,6 +1760,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
};
|
||||
|
||||
const updated = await db.transaction(async (tx) => {
|
||||
await notifyRunDispatch(tx, { companyId: scheduled.run.companyId });
|
||||
const row = await tx
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
@@ -1901,6 +1912,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
finalizeAiConnectionBusyDeferral,
|
||||
finalizeWorkspaceBusyDeferral,
|
||||
promoteDueScheduledRetries,
|
||||
nextScheduledRetryAt,
|
||||
retryScheduledRetryNow,
|
||||
|
||||
scheduleBoundedRetry: async (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js";
|
||||
import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
@@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
onBlocked: (reason, message) => { wait = { reason, message }; },
|
||||
});
|
||||
if (!admitted) {
|
||||
await db.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id);
|
||||
}
|
||||
|
||||
async function resumeExecutionWaitComments() {
|
||||
async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
await reconcileTaskChatRetryReceipts(db);
|
||||
await reconcileTaskChatRetryReceipts(db, scope);
|
||||
const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
|
||||
.from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
|
||||
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
|
||||
notInArray(issues.status, ["done", "cancelled"])))
|
||||
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
|
||||
for (const retry of retries) {
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id));
|
||||
await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"));
|
||||
await resumeTaskRetry(retry.companyId, retry.id).catch(err => {
|
||||
scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred");
|
||||
});
|
||||
}
|
||||
const waits = await db.select({ wake: agentWakeupRequests })
|
||||
.from(agentWakeupRequests)
|
||||
@@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
|
||||
executionBlockerPredicate(),
|
||||
))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
@@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
|
||||
if (canContinueCancelledRun(run)) {
|
||||
await resumeSavedComments(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
|
||||
});
|
||||
continue;
|
||||
}
|
||||
await resumeRemoteStopComments(run, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
@@ -2448,6 +2449,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await notifyRunDispatch(tx, { companyId: run.companyId });
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
ConfigurationIncompleteFailure,
|
||||
createHeartbeatRunPreparation,
|
||||
resolveExecutionRunAdapterConfig,
|
||||
readHeartbeatWakeComments,
|
||||
} from "./run-preparation.js";
|
||||
|
||||
// Existing integration suites exercise the real database and dispatch path.
|
||||
@@ -38,6 +39,12 @@ function databaseWithResults(...results: unknown[][]) {
|
||||
return { db: { select } as unknown as Db, select, scopes };
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>(complete => { resolve = complete; });
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
const emptyRoutineContext = { routineId: null, env: null, responsibleUserId: null };
|
||||
|
||||
describe("heartbeat run preparation module", () => {
|
||||
@@ -120,6 +127,42 @@ describe("heartbeat run preparation module", () => {
|
||||
expect(database.select).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("scopes one selected-comment snapshot for both wake consumers", async () => {
|
||||
const database = databaseWithResults([{ id: "comment-one", body: "selected wake" }]);
|
||||
expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", ["comment-one"]))
|
||||
.toEqual([{ id: "comment-one", body: "selected wake" }]);
|
||||
expect(database.scopes[0]).toEqual(expect.arrayContaining(["company-one", "issue-one", "comment-one"]));
|
||||
expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", [])).toEqual([]);
|
||||
expect(database.select).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("settles independent binding checks before surfacing a failure or resolving values", async () => {
|
||||
const projectCheck = deferred<[]>();
|
||||
const firstFailure = new Error("agent binding check failed");
|
||||
const secretsSvc = {
|
||||
resolveAdapterConfigForRuntime: vi.fn(), resolveEnvBindings: vi.fn(),
|
||||
collectMissingRuntimeBindings: vi.fn(async (_company, _env, context) => {
|
||||
if (context.consumerType === "agent") throw firstFailure;
|
||||
return context.consumerType === "project" ? projectCheck.promise : [];
|
||||
}),
|
||||
collectMissingAdapterConfigRuntimeBindings: vi.fn(async () => []),
|
||||
};
|
||||
let settled = false;
|
||||
const opening = resolveExecutionRunAdapterConfig({
|
||||
companyId: "company-one", agentId: "agent-one", projectId: "project-one",
|
||||
environmentId: "environment-one", environmentEnv: { SHARED: "environment" },
|
||||
executionRunConfig: { env: { SHARED: "agent" } }, projectEnv: { SHARED: "project" }, secretsSvc,
|
||||
});
|
||||
void opening.then(() => { settled = true; }, () => { settled = true; });
|
||||
await vi.waitFor(() => expect(secretsSvc.collectMissingRuntimeBindings).toHaveBeenCalledTimes(3));
|
||||
expect(secretsSvc.collectMissingAdapterConfigRuntimeBindings).toHaveBeenCalledOnce();
|
||||
expect(settled).toBe(false);
|
||||
expect(secretsSvc.resolveAdapterConfigForRuntime).not.toHaveBeenCalled();
|
||||
expect(secretsSvc.resolveEnvBindings).not.toHaveBeenCalled();
|
||||
projectCheck.resolve([]);
|
||||
await expect(opening).rejects.toBe(firstFailure);
|
||||
});
|
||||
|
||||
it("keeps missing scoped credentials as a pre-dispatch failure", async () => {
|
||||
const secretsSvc = {
|
||||
resolveAdapterConfigForRuntime: vi.fn(),
|
||||
|
||||
@@ -394,76 +394,31 @@ export async function resolveExecutionRunAdapterConfig(input: {
|
||||
// no binding before resolving any secret value. Missing bindings short-circuit
|
||||
// to a configuration-incomplete blocker routed to a human owner instead of a
|
||||
// dispatched-then-failed run (which previously surfaced as opaque setup_failed).
|
||||
if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") {
|
||||
const missingBindings: MissingRuntimeBinding[] = [];
|
||||
if (environmentEnv && input.environmentId) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingRuntimeBindings(
|
||||
input.companyId,
|
||||
environmentEnv,
|
||||
{
|
||||
consumerType: "environment",
|
||||
consumerId: input.environmentId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
}
|
||||
const collectMissing = input.secretsSvc.collectMissingRuntimeBindings;
|
||||
if (typeof collectMissing === "function") {
|
||||
const checks: Array<() => Promise<MissingRuntimeBinding[]>> = [];
|
||||
const checkEnv = (env: unknown, consumerType: "environment" | "agent" | "project" | "routine", consumerId: string) => {
|
||||
checks.push(() => collectMissing(input.companyId, env, {
|
||||
consumerType, consumerId, responsibleUserId: input.responsibleUserId ?? null,
|
||||
}));
|
||||
};
|
||||
if (environmentEnv && input.environmentId) checkEnv(environmentEnv, "environment", input.environmentId);
|
||||
if (input.agentId) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingRuntimeBindings(
|
||||
input.companyId,
|
||||
parseObject(executionRunConfig.env),
|
||||
{
|
||||
consumerType: "agent",
|
||||
consumerId: input.agentId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
if (
|
||||
typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings ===
|
||||
"function"
|
||||
) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings(
|
||||
input.companyId,
|
||||
executionRunConfig,
|
||||
input.adapterType ?? null,
|
||||
{
|
||||
consumerType: "agent",
|
||||
consumerId: input.agentId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
checkEnv(parseObject(executionRunConfig.env), "agent", input.agentId);
|
||||
const collectAdapterMissing = input.secretsSvc.collectMissingAdapterConfigRuntimeBindings;
|
||||
if (typeof collectAdapterMissing === "function") {
|
||||
checks.push(() => collectAdapterMissing(input.companyId, executionRunConfig, input.adapterType ?? null, {
|
||||
consumerType: "agent", consumerId: input.agentId!, responsibleUserId: input.responsibleUserId ?? null,
|
||||
}));
|
||||
}
|
||||
}
|
||||
if (projectEnv && input.projectId) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingRuntimeBindings(
|
||||
input.companyId,
|
||||
projectEnv,
|
||||
{
|
||||
consumerType: "project",
|
||||
consumerId: input.projectId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
}
|
||||
if (routineEnv && input.routineId) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingRuntimeBindings(
|
||||
input.companyId,
|
||||
routineEnv,
|
||||
{
|
||||
consumerType: "routine",
|
||||
consumerId: input.routineId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
if (projectEnv && input.projectId) checkEnv(projectEnv, "project", input.projectId);
|
||||
if (routineEnv && input.routineId) checkEnv(routineEnv, "routine", input.routineId);
|
||||
const results = await Promise.allSettled(checks.map(check => Promise.resolve().then(check)));
|
||||
const missingBindings: MissingRuntimeBinding[] = [];
|
||||
for (const result of results) {
|
||||
if (result.status === "rejected") throw result.reason;
|
||||
missingBindings.push(...result.value);
|
||||
}
|
||||
if (requiredScopedEnvBinding) {
|
||||
const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys);
|
||||
@@ -2244,12 +2199,45 @@ export async function attestReviewedExternalChatRun(input: {
|
||||
throw new Error("reviewed_chat_execution_binding_not_ready");
|
||||
}
|
||||
|
||||
export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) {
|
||||
if (commentIds.length === 0) return [];
|
||||
return db
|
||||
.select({
|
||||
id: issueComments.id,
|
||||
companyId: issueComments.companyId,
|
||||
issueId: issueComments.issueId,
|
||||
body: issueComments.body,
|
||||
origin: issueComments.origin,
|
||||
authorType: issueComments.authorType,
|
||||
authorAgentId: issueComments.authorAgentId,
|
||||
authorUserId: issueComments.authorUserId,
|
||||
presentation: issueComments.presentation,
|
||||
metadata: issueComments.metadata,
|
||||
deletedAt: issueComments.deletedAt,
|
||||
deletedByType: issueComments.deletedByType,
|
||||
deletedByAgentId: issueComments.deletedByAgentId,
|
||||
deletedByUserId: issueComments.deletedByUserId,
|
||||
deletedByRunId: issueComments.deletedByRunId,
|
||||
sourceTrust: issueComments.sourceTrust,
|
||||
createdAt: issueComments.createdAt,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.companyId, companyId),
|
||||
issueId ? eq(issueComments.issueId, issueId) : undefined,
|
||||
inArray(issueComments.id, commentIds),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export async function buildPaperclipWakePayload(input: {
|
||||
db: Db;
|
||||
companyId: string;
|
||||
agentId?: string | null;
|
||||
runId?: string | null;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
wakeCommentRows?: Awaited<ReturnType<typeof readHeartbeatWakeComments>>;
|
||||
continuationSummary?: {
|
||||
key: string;
|
||||
title: string | null;
|
||||
@@ -2314,36 +2302,9 @@ export async function buildPaperclipWakePayload(input: {
|
||||
)
|
||||
return null;
|
||||
|
||||
const commentRows =
|
||||
commentIds.length === 0
|
||||
? []
|
||||
: await input.db
|
||||
.select({
|
||||
id: issueComments.id,
|
||||
issueId: issueComments.issueId,
|
||||
body: issueComments.body,
|
||||
origin: issueComments.origin,
|
||||
authorType: issueComments.authorType,
|
||||
authorAgentId: issueComments.authorAgentId,
|
||||
authorUserId: issueComments.authorUserId,
|
||||
presentation: issueComments.presentation,
|
||||
metadata: issueComments.metadata,
|
||||
deletedAt: issueComments.deletedAt,
|
||||
deletedByType: issueComments.deletedByType,
|
||||
deletedByAgentId: issueComments.deletedByAgentId,
|
||||
deletedByUserId: issueComments.deletedByUserId,
|
||||
deletedByRunId: issueComments.deletedByRunId,
|
||||
sourceTrust: issueComments.sourceTrust,
|
||||
createdAt: issueComments.createdAt,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.companyId, input.companyId),
|
||||
issueId ? eq(issueComments.issueId, issueId) : undefined,
|
||||
inArray(issueComments.id, commentIds),
|
||||
),
|
||||
);
|
||||
const commentRows = input.wakeCommentRows
|
||||
? input.wakeCommentRows.filter(row => row.companyId === input.companyId && (!issueId || row.issueId === issueId) && commentIds.includes(row.id))
|
||||
: await readHeartbeatWakeComments(input.db, input.companyId, issueId, commentIds);
|
||||
|
||||
const commentsById = new Map(
|
||||
commentRows.map((comment) => [comment.id, comment]),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { notifyIssueReviewState } from "../issue-review-state-notifications.js";
|
||||
import { insertIssueComment } from "../issue-comment-writes.js";
|
||||
@@ -595,6 +596,7 @@ export function createHeartbeatScheduling(
|
||||
try {
|
||||
await db.transaction(async (tx) => {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, claimed.companyId);
|
||||
const released = await tx.update(issues)
|
||||
.set({ monitorWakeRequestedAt: null, updatedAt: input.now })
|
||||
.where(issueMonitorClaimCondition(claimed)).returning({ id: issues.id });
|
||||
@@ -628,6 +630,7 @@ export function createHeartbeatScheduling(
|
||||
// failed, and never clobber a monitor that was replaced in the meantime.
|
||||
await db.transaction(async tx => {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, claimed.companyId);
|
||||
await tx.update(issues).set({ monitorWakeRequestedAt: null, updatedAt: input.now })
|
||||
.where(issueMonitorClaimCondition(claimed));
|
||||
});
|
||||
@@ -827,6 +830,7 @@ export function createHeartbeatScheduling(
|
||||
}
|
||||
|
||||
const consumed = await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, claimed.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
return tx.update(issues)
|
||||
.set({
|
||||
@@ -977,6 +981,7 @@ export function createHeartbeatScheduling(
|
||||
|
||||
const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000);
|
||||
const claimed = await db.transaction(async (tx) => {
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
const [updated] = await tx
|
||||
.update(issues)
|
||||
@@ -1050,6 +1055,7 @@ export function createHeartbeatScheduling(
|
||||
for (const due of dueMonitors) {
|
||||
if (!canContinue()) break;
|
||||
const claimed = await db.transaction(async (tx) => {
|
||||
await notifyStatusCardIssueChange(tx, due.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
const [updated] = await tx
|
||||
.update(issues)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db";
|
||||
import type { Db, agents, agentTaskSessions, heartbeatRuns } from "@paperclipai/db";
|
||||
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
|
||||
import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
|
||||
import { parseObject } from "../../adapters/utils.js";
|
||||
@@ -15,8 +15,9 @@ import { getTaskPlanContext } from "../task-plan-context.js";
|
||||
import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js";
|
||||
import type { TrustPresetResolution } from "../trust-preset-resolver.js";
|
||||
import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js";
|
||||
import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
|
||||
import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, readHeartbeatWakeComments, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
|
||||
import { buildPaperclipTaskMarkdown } from "./task-markdown.js";
|
||||
import { deriveCommentId, extractWakeCommentIds } from "../../modules/run-dispatch/index.js";
|
||||
import { workspacePreferenceService } from "../workspace-preferences.js";
|
||||
|
||||
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
|
||||
@@ -30,8 +31,6 @@ export interface HeartbeatTaskContextInput {
|
||||
"id" | "identifier" | "title" | "status" | "priority" | "workMode" |
|
||||
"conversationAgentId" | "description" | "projectId"> | null;
|
||||
issueContext: IssueContext | null;
|
||||
wakeCommentContext: Pick<typeof issueComments.$inferSelect,
|
||||
"id" | "body" | "sourceTrust" | "presentation" | "metadata"> | null;
|
||||
context: Record<string, unknown>;
|
||||
taskSession: Pick<typeof agentTaskSessions.$inferSelect, "lastRunId"> | null;
|
||||
};
|
||||
@@ -53,35 +52,9 @@ function readNonEmptyString(value: unknown): string | null {
|
||||
/** Rebuilds task prompt fields on the run's context; acquires no execution resources. */
|
||||
export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) {
|
||||
const { run, agent } = input;
|
||||
const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task;
|
||||
const { issueRef, issueContext, context, taskSession } = input.task;
|
||||
const { trustPreset, experimentalInstanceSettings } = input.policy;
|
||||
const { issuesSvc, getPinnedSkillTestContext } = input.services;
|
||||
const continuationSummary = issueRef && !isConversation(issueContext)
|
||||
? await getIssueContinuationSummaryDocument(db, issueRef.id)
|
||||
: null;
|
||||
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
|
||||
const safeContinuationSummary =
|
||||
continuationSummary && !exposeLowTrustRaw
|
||||
? redactQuarantinedBodyForHigherTrust(continuationSummary)
|
||||
: continuationSummary;
|
||||
const safeWakeCommentContext =
|
||||
wakeCommentContext && !exposeLowTrustRaw
|
||||
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
|
||||
: wakeCommentContext;
|
||||
const issueAncestors = issueRef
|
||||
? await issuesSvc.getAncestors(issueRef.id)
|
||||
: [];
|
||||
if (continuationSummary) {
|
||||
context.paperclipContinuationSummary = {
|
||||
key: safeContinuationSummary!.key,
|
||||
title: safeContinuationSummary!.title,
|
||||
body: safeContinuationSummary!.body,
|
||||
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
|
||||
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
|
||||
};
|
||||
} else {
|
||||
delete context.paperclipContinuationSummary;
|
||||
}
|
||||
const admittedScope = parseObject(context.paperclipAgentJwtScope);
|
||||
if (!input.policy.recovering || (admittedScope.kind !== "standard" && admittedScope.kind !== "skill_test")) {
|
||||
const savedIssue = parseObject(context.paperclipIssue);
|
||||
@@ -93,10 +66,58 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
|
||||
: { kind: "standard" };
|
||||
}
|
||||
const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test";
|
||||
const pinnedSkillTestContext =
|
||||
isSkillTest && issueRef
|
||||
? await getPinnedSkillTestContext(agent.companyId, issueRef.id)
|
||||
: null;
|
||||
const wakeCommentId = deriveCommentId(context, null);
|
||||
const commentIds = [...new Set([...extractWakeCommentIds(context), ...(wakeCommentId ? [wakeCommentId] : [])])];
|
||||
const [summary, ancestors, pinnedSkillTestRead, wakeComments] = await Promise.allSettled([
|
||||
issueRef && !isConversation(issueContext) ? getIssueContinuationSummaryDocument(db, issueRef.id) : Promise.resolve(null),
|
||||
issueRef ? issuesSvc.getAncestors(issueRef.id) : Promise.resolve([]),
|
||||
isSkillTest && issueRef ? getPinnedSkillTestContext(agent.companyId, issueRef.id) : Promise.resolve(null),
|
||||
readHeartbeatWakeComments(db, agent.companyId, readNonEmptyString(context.issueId), commentIds),
|
||||
]);
|
||||
if (summary.status === "rejected") throw summary.reason;
|
||||
if (ancestors.status === "rejected") throw ancestors.reason;
|
||||
if (pinnedSkillTestRead.status === "rejected") throw pinnedSkillTestRead.reason;
|
||||
if (wakeComments.status === "rejected") throw wakeComments.reason;
|
||||
const wakeCommentRows = wakeComments.value;
|
||||
const selectedWakeComment = issueRef && wakeCommentId ? wakeCommentRows.find(comment => comment.id === wakeCommentId && comment.issueId === issueRef.id) ?? null : null;
|
||||
const wakeCommentContext = selectedWakeComment ? {
|
||||
id: selectedWakeComment.id,
|
||||
body: selectedWakeComment.deletedAt ? "" : selectedWakeComment.body,
|
||||
authorType: selectedWakeComment.authorType,
|
||||
authorAgentId: selectedWakeComment.authorAgentId,
|
||||
authorUserId: selectedWakeComment.authorUserId,
|
||||
presentation: selectedWakeComment.deletedAt ? null : selectedWakeComment.presentation,
|
||||
metadata: selectedWakeComment.deletedAt ? null : selectedWakeComment.metadata,
|
||||
deletedAt: selectedWakeComment.deletedAt,
|
||||
deletedByType: selectedWakeComment.deletedByType,
|
||||
deletedByAgentId: selectedWakeComment.deletedByAgentId,
|
||||
deletedByUserId: selectedWakeComment.deletedByUserId,
|
||||
deletedByRunId: selectedWakeComment.deletedByRunId,
|
||||
sourceTrust: selectedWakeComment.sourceTrust,
|
||||
} : null;
|
||||
const continuationSummary = summary.value;
|
||||
const issueAncestors = ancestors.value;
|
||||
const pinnedSkillTestContext = pinnedSkillTestRead.value;
|
||||
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
|
||||
const safeContinuationSummary =
|
||||
continuationSummary && !exposeLowTrustRaw
|
||||
? redactQuarantinedBodyForHigherTrust(continuationSummary)
|
||||
: continuationSummary;
|
||||
const safeWakeCommentContext =
|
||||
wakeCommentContext && !exposeLowTrustRaw
|
||||
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
|
||||
: wakeCommentContext;
|
||||
if (continuationSummary) {
|
||||
context.paperclipContinuationSummary = {
|
||||
key: safeContinuationSummary!.key,
|
||||
title: safeContinuationSummary!.title,
|
||||
body: safeContinuationSummary!.body,
|
||||
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
|
||||
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
|
||||
};
|
||||
} else {
|
||||
delete context.paperclipContinuationSummary;
|
||||
}
|
||||
if (isSkillTest && !pinnedSkillTestContext) {
|
||||
throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", {
|
||||
configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" },
|
||||
@@ -132,6 +153,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
|
||||
agentId: agent.id,
|
||||
runId: run.id,
|
||||
contextSnapshot: context,
|
||||
wakeCommentRows,
|
||||
continuationSummary,
|
||||
issueSummary: issueRef
|
||||
? {
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js";
|
||||
@@ -625,11 +628,14 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
|
||||
}
|
||||
const now = new Date();
|
||||
const [updated] = await db.transaction(async tx => {
|
||||
if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch);
|
||||
if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx);
|
||||
if (patch.enableWorktreeRunExecution !== undefined) {
|
||||
await notifyRunDispatch(tx);
|
||||
await notifyDeferredExecutionEligibility(tx);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyExternalAgentSettlement(tx);
|
||||
await notifyInstructionCommit(tx);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import type { PgInsertValue } from "drizzle-orm/pg-core";
|
||||
import { issueComments, type Db } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
@@ -6,6 +8,9 @@ type WriteDb = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
|
||||
export function insertIssueComment(db: WriteDb, input: PgInsertValue<typeof issueComments> & { companyId: string }) {
|
||||
return db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, input.companyId);
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId,
|
||||
issueId: typeof input.issueId === "string" ? input.issueId : undefined });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`);
|
||||
return tx.insert(issueComments).values(input).returning();
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import type { PgColumn } from "drizzle-orm/pg-core";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -127,6 +128,7 @@ export async function rekeyCompanyIssueIdentifiers(
|
||||
// overload, which returns NULL for a non-pattern argument.
|
||||
const tailStart = sql`${fromPrefix.length + 1}::int`;
|
||||
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const rekeyedIssues = await tx
|
||||
.update(issues)
|
||||
.set({ identifier: sql`${toPrefix} || substring(${issues.identifier} from ${tailStart})` })
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { writeExecutionControl } from "./execution-control-notifications.js";
|
||||
@@ -322,6 +324,8 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
): Promise<IssueRecoveryAction> {
|
||||
try {
|
||||
const created = await db.transaction(async (tx) => {
|
||||
await notifyAttentionRetentionWork(tx, { companyId: input.companyId });
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation);
|
||||
@@ -413,6 +417,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
Math.min(nextAttemptCount, effectiveMaxAttempts),
|
||||
);
|
||||
const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
if ((!input.preserveExistingOwner && existing.cause !== input.cause)
|
||||
|| (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
|
||||
@@ -474,6 +479,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
return toReadModel(exhausted);
|
||||
}
|
||||
const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
if ((!input.preserveExistingOwner && existing.cause !== input.cause)
|
||||
|| (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
|
||||
@@ -548,6 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
|
||||
try {
|
||||
const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
return tx
|
||||
@@ -595,6 +602,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions)
|
||||
.where(and(...predicates)).limit(1);
|
||||
if (!pending.length) return [];
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
return tx
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
/** An issue hint is reserved for local facts such as comments. Lifecycle,
|
||||
* graph, run and eligibility changes retain company scope because their
|
||||
* consequences can include ancestors and shared execution references. */
|
||||
export type IssueRecoveryScope = { companyId: string; issueId?: string };
|
||||
|
||||
export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) {
|
||||
const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
|
||||
const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*";
|
||||
for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
|
||||
DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) {
|
||||
await notifyDeliveryWork(transaction, queue,
|
||||
queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint);
|
||||
}
|
||||
}
|
||||
Loaded 100 of 201 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user