mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(heartbeat): run one dispatch pass per agent, back off refused claims, and enforce the run limit in the claim
Triggers during a pass fold into one follow-up pass instead of stacking past the 30 s stale lock. Each claim checks the agent's running count against maxConcurrentRuns under the agent admission lock, so overlapping passes and a second server cannot run an agent past its limit. A refused claim waits a bounded backoff before an event-driven pass offers it again; sweeps and a person's runs ignore it. The pass reads only the columns it needs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
426e8a35dc
commit
68e64a2de9
8 files changed
+675
-173
No files matched your search
@@ -1,30 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { withAgentStartLock } from "../services/agent-start-lock.ts";
|
||||
|
||||
describe("heartbeat agent start lock", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("does not let a stale start lock freeze later queued-run starts", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const agentId = randomUUID();
|
||||
const firstStart = vi.fn(() => new Promise<void>(() => undefined));
|
||||
const secondStart = vi.fn(async () => "started");
|
||||
|
||||
void withAgentStartLock(agentId, firstStart);
|
||||
await Promise.resolve();
|
||||
expect(firstStart).toHaveBeenCalledTimes(1);
|
||||
|
||||
const secondStartResult = withAgentStartLock(agentId, secondStart);
|
||||
await Promise.resolve();
|
||||
expect(secondStart).not.toHaveBeenCalled();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
|
||||
await expect(secondStartResult).resolves.toBe("started");
|
||||
expect(secondStart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,237 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agentWakeupRequests, agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues,
|
||||
} from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
import { runningProcesses } from "../adapters/index.ts";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { isAgentDispatchActive } from "../services/agent-start-lock.js";
|
||||
|
||||
const held = vi.hoisted(() => {
|
||||
const state = {
|
||||
release: () => undefined as void,
|
||||
released: Promise.resolve(),
|
||||
reset() {
|
||||
state.released = new Promise<void>((resolve) => { state.release = resolve; });
|
||||
},
|
||||
};
|
||||
return state;
|
||||
})
|
||||
|
||||
const passes = vi.hoisted(() => ({ started: 0, active: 0, peak: 0 }));
|
||||
|
||||
vi.mock("../adapters/index.ts", async () => {
|
||||
const actual = await vi.importActual<typeof import("../adapters/index.ts")>("../adapters/index.ts");
|
||||
return {
|
||||
...actual,
|
||||
getServerAdapter: vi.fn(() => ({
|
||||
supportsLocalAgentJwt: false,
|
||||
execute: async () => {
|
||||
await held.released;
|
||||
return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "done", provider: "test", model: "test" };
|
||||
},
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../services/agent-start-lock.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../services/agent-start-lock.js")>("../services/agent-start-lock.js");
|
||||
const measure = async <T>(pass: () => Promise<T>) => {
|
||||
passes.started += 1;
|
||||
passes.active += 1;
|
||||
passes.peak = Math.max(passes.peak, passes.active);
|
||||
try {
|
||||
return await pass();
|
||||
} finally {
|
||||
passes.active -= 1;
|
||||
}
|
||||
};
|
||||
return {
|
||||
...actual,
|
||||
dispatchForAgent: (agentId: string, spec: Parameters<typeof actual.dispatchForAgent>[1]) =>
|
||||
actual.dispatchForAgent(agentId, { ...spec, run: (request, signal) => measure(() => spec.run(request, signal)) }),
|
||||
};
|
||||
});
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const suite = support.supported ? describe : describe.skip;
|
||||
|
||||
const CAP = 4;
|
||||
const UNCLAIMABLE = 12;
|
||||
const CLAIMABLE = 3 * CAP;
|
||||
const TRIGGERS = 40;
|
||||
|
||||
suite("queued-run dispatch under a trigger burst", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let companyId = "", userId = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("paperclip-queued-dispatch-stacking-");
|
||||
db = createDb(temporary.connectionString);
|
||||
await db.execute(sql`create table running_peaks (agent_id uuid not null, running int not null)`);
|
||||
await db.execute(sql`create function record_running_peak() returns trigger language plpgsql as $$
|
||||
begin
|
||||
insert into running_peaks select new.agent_id, count(*) from heartbeat_runs where agent_id = new.agent_id and status = 'running';
|
||||
return new;
|
||||
end $$`);
|
||||
await db.execute(sql`create trigger record_running_peak after update of status on heartbeat_runs
|
||||
for each row when (new.status = 'running') execute function record_running_peak()`);
|
||||
}, 60_000);
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
held.release();
|
||||
runningProcesses.clear();
|
||||
await db.execute(sql`truncate table ${companies} cascade`);
|
||||
});
|
||||
|
||||
afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); });
|
||||
|
||||
async function seedAgent() {
|
||||
userId = randomUUID();
|
||||
companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name: "Board", email: `${userId}@example.test`,
|
||||
emailVerified: true, createdAt: new Date(), updatedAt: new Date() });
|
||||
await db.insert(companies).values({ id: companyId, name: "Burst", issuePrefix: `B${companyId.replaceAll("-", "").slice(0, 6)}`,
|
||||
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: userId });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Codie", role: "engineer", status: "idle",
|
||||
adapterType: "codex_local", adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { enabled: true, wakeOnDemand: true, maxConcurrentRuns: CAP } }, permissions: {} });
|
||||
return agentId;
|
||||
}
|
||||
|
||||
async function task(agentId: string) {
|
||||
const [row] = await db.insert(issues).values({ companyId, title: "Continue the work", status: "in_progress", assigneeAgentId: agentId }).returning();
|
||||
return row!;
|
||||
}
|
||||
|
||||
async function finishedRun(agentId: string, issueId: string, minutesAgo: number) {
|
||||
const finishedAt = new Date(Date.now() - minutesAgo * 60_000);
|
||||
const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId, invocationSource: "assignment",
|
||||
status: "failed", errorCode: "server_shutdown_interrupted", startedAt: new Date(finishedAt.getTime() - 60_000), finishedAt,
|
||||
createdAt: new Date(finishedAt.getTime() - 120_000), contextSnapshot: { issueId, wakeReason: "issue_assigned" } }).returning();
|
||||
return row!;
|
||||
}
|
||||
|
||||
async function queuedContinuation(agentId: string, input: { minutesAgo: number; wakeReason: string; sourceIssueId?: string }) {
|
||||
const issue = await task(agentId);
|
||||
const source = await finishedRun(agentId, input.sourceIssueId ?? issue.id, input.minutesAgo + 5);
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", triggerDetail: "system",
|
||||
reason: input.wakeReason, status: "queued", requestedByActorType: "system", requestedByActorId: null,
|
||||
payload: { issueId: issue.id } }).returning();
|
||||
const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "automation",
|
||||
triggerDetail: "system", status: "queued", retryOfRunId: source.id, wakeupRequestId: wake!.id,
|
||||
createdAt: new Date(Date.now() - input.minutesAgo * 60_000),
|
||||
contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: input.wakeReason, previousRunId: source.id,
|
||||
source: "issue.assigned_work_invariant", paperclipWorkspace: { notes: "x".repeat(16_000) } } }).returning();
|
||||
await db.update(agentWakeupRequests).set({ runId: row!.id }).where(eq(agentWakeupRequests.id, wake!.id));
|
||||
return row!;
|
||||
}
|
||||
|
||||
const peakRunning = async (agentId: string) => Number((await db.execute(sql`select coalesce(max(running), 0) as peak
|
||||
from running_peaks where agent_id = ${agentId}::uuid`))[0]!.peak);
|
||||
|
||||
it("runs one pass at a time for a burst of triggers, offers a refused run once, and never runs past the agent's limit", async () => {
|
||||
held.reset();
|
||||
passes.started = 0; passes.active = 0; passes.peak = 0;
|
||||
const agentId = await seedAgent();
|
||||
const otherTask = await task(agentId);
|
||||
const unclaimable = new Set<string>();
|
||||
for (let index = 0; index < UNCLAIMABLE; index += 1) {
|
||||
unclaimable.add((await queuedContinuation(agentId, { minutesAgo: 180 - index, wakeReason: "issue_continuation_needed",
|
||||
sourceIssueId: otherTask.id })).id);
|
||||
}
|
||||
for (let index = 0; index < CLAIMABLE; index += 1) {
|
||||
await queuedContinuation(agentId, { minutesAgo: 30 - index, wakeReason: "issue_assigned" });
|
||||
}
|
||||
|
||||
const realNow = Date.now.bind(Date);
|
||||
let skewMs = 0;
|
||||
vi.spyOn(Date, "now").mockImplementation(() => realNow() + skewMs);
|
||||
const offers = new Map<string, number>();
|
||||
let inFlightClaims = 0;
|
||||
let peakInFlightClaims = 0;
|
||||
const triggers: Array<Promise<unknown>> = [];
|
||||
let burstFired = false;
|
||||
const heartbeat = heartbeatService(db, {
|
||||
beforeChatControlRecoveryCheck: async ({ runId, stage }) => {
|
||||
if (stage !== "claim") return;
|
||||
offers.set(runId, (offers.get(runId) ?? 0) + 1);
|
||||
inFlightClaims += 1;
|
||||
peakInFlightClaims = Math.max(peakInFlightClaims, inFlightClaims);
|
||||
try {
|
||||
if (!burstFired) {
|
||||
burstFired = true;
|
||||
for (let index = 0; index < TRIGGERS; index += 1) {
|
||||
skewMs += 31_000;
|
||||
triggers.push(heartbeat.resumeQueuedRunsForAgent(companyId, agentId));
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
skewMs = 0;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
} finally {
|
||||
inFlightClaims -= 1;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
|
||||
await Promise.all(triggers);
|
||||
const refusedOffers = [...unclaimable].reduce((sum, id) => sum + (offers.get(id) ?? 0), 0);
|
||||
expect({
|
||||
passes: passes.started,
|
||||
peakConcurrentPasses: passes.peak,
|
||||
peakInFlightClaims,
|
||||
refusedOffersPerTrigger: refusedOffers / (TRIGGERS + 1),
|
||||
peakRunning: await peakRunning(agentId),
|
||||
}).toEqual({
|
||||
passes: 2,
|
||||
peakConcurrentPasses: 1,
|
||||
peakInFlightClaims: 1,
|
||||
refusedOffersPerTrigger: UNCLAIMABLE / (TRIGGERS + 1),
|
||||
peakRunning: CAP,
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(passes.active).toBe(0);
|
||||
expect(isAgentDispatchActive(agentId)).toBe(false);
|
||||
}, { timeout: 10_000 });
|
||||
|
||||
} finally {
|
||||
held.release();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it.each(["declined", "rejected"] as const)("waits a bounded time before offering a %s run again, and offers it again once that time passes", async (refusal) => {
|
||||
const agentId = await seedAgent();
|
||||
if (refusal === "rejected") {
|
||||
await db.update(companies).set({ defaultResponsibleUserId: null }).where(eq(companies.id, companyId));
|
||||
await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId));
|
||||
}
|
||||
const otherTask = await task(agentId);
|
||||
const refused = await queuedContinuation(agentId, { minutesAgo: 180, wakeReason: "issue_continuation_needed", sourceIssueId: otherTask.id });
|
||||
const warn = vi.spyOn(logger, "warn");
|
||||
const offersOf = () => warn.mock.calls.filter(([fields, message]) => typeof message === "string"
|
||||
&& message.startsWith("long-queued heartbeat run was offered a free slot")
|
||||
&& (fields as { runId?: string }).runId === refused.id).length;
|
||||
const heartbeat = heartbeatService(db);
|
||||
try {
|
||||
for (let index = 0; index < 10; index += 1) await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
|
||||
expect(offersOf()).toBe(1);
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
|
||||
expect(offersOf()).toBe(2);
|
||||
}, { timeout: 20_000, interval: 1_000 });
|
||||
} finally {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
AGENT_DISPATCH_PASS_DEADLINE_MS,
|
||||
dispatchForAgent,
|
||||
isAgentDispatchActive,
|
||||
type AgentDispatchPass,
|
||||
} from "./agent-start-lock.js";
|
||||
|
||||
function gate() {
|
||||
let open!: () => void;
|
||||
const opened = new Promise<void>((resolve) => { open = resolve; });
|
||||
return { opened, open };
|
||||
}
|
||||
|
||||
function meter() {
|
||||
const stats = { passes: 0, active: 0, peak: 0, requests: [] as boolean[] };
|
||||
const spec = (attach: boolean, body: (signal: AbortSignal) => Promise<void>): AgentDispatchPass<boolean, number> => ({
|
||||
request: attach,
|
||||
idle: -1,
|
||||
merge: (current, next) => current || next,
|
||||
async run(request, signal) {
|
||||
stats.passes += 1;
|
||||
stats.active += 1;
|
||||
stats.peak = Math.max(stats.peak, stats.active);
|
||||
stats.requests.push(request);
|
||||
const pass = stats.passes;
|
||||
try {
|
||||
await body(signal);
|
||||
} finally {
|
||||
stats.active -= 1;
|
||||
}
|
||||
return { value: pass };
|
||||
},
|
||||
});
|
||||
return { stats, spec };
|
||||
}
|
||||
|
||||
describe("agent dispatch passes", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("runs one pass per agent and folds every trigger that arrives during it into one follow-up pass", async () => {
|
||||
const agentId = randomUUID();
|
||||
const { stats, spec } = meter();
|
||||
const first = gate();
|
||||
const firstPass = dispatchForAgent(agentId, spec(false, () => first.opened));
|
||||
const burst = Array.from({ length: 50 }, (_, index) =>
|
||||
dispatchForAgent(agentId, spec(index === 7, async () => undefined)));
|
||||
expect(stats).toMatchObject({ passes: 1, peak: 1 });
|
||||
first.open();
|
||||
expect(await firstPass).toBe(1);
|
||||
expect(new Set(await Promise.all(burst))).toEqual(new Set([2]));
|
||||
expect(stats).toMatchObject({ passes: 2, peak: 1, requests: [false, true] });
|
||||
expect(isAgentDispatchActive(agentId)).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps a second pass out while a slow pass runs past the old 30 second stale limit", async () => {
|
||||
vi.useFakeTimers();
|
||||
const agentId = randomUUID();
|
||||
const { stats, spec } = meter();
|
||||
const slow = gate();
|
||||
const firstPass = dispatchForAgent(agentId, spec(false, () => slow.opened));
|
||||
const triggers: Array<Promise<number>> = [];
|
||||
for (let second = 0; second < 45; second += 1) {
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
triggers.push(dispatchForAgent(agentId, spec(false, async () => undefined)));
|
||||
}
|
||||
expect(stats).toMatchObject({ passes: 1, peak: 1 });
|
||||
slow.open();
|
||||
await firstPass;
|
||||
await Promise.all(triggers);
|
||||
expect(stats).toMatchObject({ passes: 2, peak: 1 });
|
||||
});
|
||||
|
||||
it("stops a hung pass at its deadline and lets the waiting trigger run without overlapping it", async () => {
|
||||
vi.useFakeTimers();
|
||||
const agentId = randomUUID();
|
||||
const { stats, spec } = meter();
|
||||
let hungSignal: AbortSignal | null = null;
|
||||
void dispatchForAgent(agentId, spec(false, (signal) => {
|
||||
hungSignal = signal;
|
||||
return new Promise<void>(() => undefined);
|
||||
}));
|
||||
const waiting = dispatchForAgent(agentId, spec(false, async () => undefined));
|
||||
await vi.advanceTimersByTimeAsync(AGENT_DISPATCH_PASS_DEADLINE_MS - 1);
|
||||
expect(stats.passes).toBe(1);
|
||||
expect(hungSignal!.aborted).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(hungSignal!.aborted).toBe(true);
|
||||
expect(await waiting).toBe(2);
|
||||
expect(stats.passes).toBe(2);
|
||||
expect(isAgentDispatchActive(agentId)).toBe(false);
|
||||
expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(3);
|
||||
});
|
||||
|
||||
it("does not wait on itself when a pass triggers its own agent", async () => {
|
||||
const agentId = randomUUID();
|
||||
const { stats, spec } = meter();
|
||||
let nested: Promise<number> | null = null;
|
||||
const pass = dispatchForAgent(agentId, spec(false, async () => {
|
||||
nested = dispatchForAgent(agentId, spec(false, async () => undefined));
|
||||
expect(await nested).toBe(-1);
|
||||
}));
|
||||
expect(await pass).toBe(1);
|
||||
await vi.waitFor(() => expect(stats.passes).toBe(2));
|
||||
expect(stats.peak).toBe(1);
|
||||
});
|
||||
|
||||
it("settles a pass's effects after the next pass may start", async () => {
|
||||
const agentId = randomUUID();
|
||||
const order: string[] = [];
|
||||
const release = gate();
|
||||
const pass = dispatchForAgent(agentId, {
|
||||
request: null, idle: null, merge: () => null,
|
||||
async run() {
|
||||
await release.opened;
|
||||
return {
|
||||
value: null,
|
||||
settle: async () => {
|
||||
order.push(`settle:${isAgentDispatchActive(agentId)}`);
|
||||
await dispatchForAgent(agentId, { request: null, idle: null, merge: () => null,
|
||||
async run() { order.push("nested"); return { value: null }; } });
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
release.open();
|
||||
await pass;
|
||||
expect(order).toEqual(["settle:false", "nested"]);
|
||||
});
|
||||
|
||||
it("passes a failed pass's error to its caller and frees the agent", async () => {
|
||||
const agentId = randomUUID();
|
||||
const { spec } = meter();
|
||||
await expect(dispatchForAgent(agentId, spec(false, async () => { throw new Error("boom"); }))).rejects.toThrow("boom");
|
||||
expect(isAgentDispatchActive(agentId)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,48 +1,90 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
|
||||
const AGENT_START_LOCK_STALE_MS = 30_000;
|
||||
const startLocksByAgent = new Map<string, { promise: Promise<void>; startedAtMs: number }>();
|
||||
export const AGENT_DISPATCH_PASS_DEADLINE_MS = 60_000;
|
||||
|
||||
async function waitForAgentStartLock(agentId: string, lock: { promise: Promise<void>; startedAtMs: number }) {
|
||||
const elapsedMs = Date.now() - lock.startedAtMs;
|
||||
const remainingMs = AGENT_START_LOCK_STALE_MS - elapsedMs;
|
||||
if (remainingMs <= 0) {
|
||||
logger.warn({ agentId, staleMs: elapsedMs }, "agent start lock stale; continuing queued-run start");
|
||||
return;
|
||||
}
|
||||
export type AgentDispatchPass<R, T> = {
|
||||
request: R;
|
||||
idle: T;
|
||||
merge(current: R, next: R): R;
|
||||
run(request: R, signal: AbortSignal): Promise<{ value: T; settle?: () => Promise<void> }>;
|
||||
track?(pass: Promise<unknown>): void;
|
||||
};
|
||||
|
||||
let timedOut = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
await Promise.race([
|
||||
lock.promise,
|
||||
new Promise<void>((resolve) => {
|
||||
timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
resolve();
|
||||
}, remainingMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
type Deferred<T> = { promise: Promise<T>; resolve(value: T): void; reject(err: unknown): void };
|
||||
type FollowUp = { request: unknown; spec: AgentDispatchPass<unknown, unknown>; done: Deferred<unknown> };
|
||||
type AgentDispatchState = { followUp: FollowUp | null };
|
||||
|
||||
if (timedOut) {
|
||||
logger.warn({ agentId, staleMs: AGENT_START_LOCK_STALE_MS }, "agent start lock timed out; continuing queued-run start");
|
||||
}
|
||||
const dispatchStates = new Map<string, AgentDispatchState>();
|
||||
const dispatchingAgent = new AsyncLocalStorage<string>();
|
||||
|
||||
function deferred<T>(): Deferred<T> {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (err: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
promise.catch(() => undefined);
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
export async function withAgentStartLock<T>(agentId: string, fn: () => Promise<T>) {
|
||||
const previous = startLocksByAgent.get(agentId);
|
||||
const waitForPrevious = previous ? waitForAgentStartLock(agentId, previous) : Promise.resolve();
|
||||
const run = waitForPrevious.then(fn);
|
||||
const marker = run.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
export function isAgentDispatchActive(agentId: string) {
|
||||
return dispatchStates.has(agentId);
|
||||
}
|
||||
|
||||
export function outsideAgentDispatch<T>(fn: () => T): T {
|
||||
return dispatchingAgent.exit(fn);
|
||||
}
|
||||
|
||||
export function dispatchForAgent<R, T>(agentId: string, spec: AgentDispatchPass<R, T>): Promise<T> {
|
||||
const state = dispatchStates.get(agentId);
|
||||
if (!state) return startPass(agentId, spec, spec.request);
|
||||
if (state.followUp) {
|
||||
state.followUp.request = spec.merge(state.followUp.request as R, spec.request);
|
||||
state.followUp.spec = spec as AgentDispatchPass<unknown, unknown>;
|
||||
} else {
|
||||
state.followUp = { request: spec.request, spec: spec as AgentDispatchPass<unknown, unknown>, done: deferred() };
|
||||
}
|
||||
if (dispatchingAgent.getStore() === agentId) return Promise.resolve(spec.idle);
|
||||
return state.followUp.done.promise as Promise<T>;
|
||||
}
|
||||
|
||||
function startPass<R, T>(agentId: string, spec: AgentDispatchPass<R, T>, request: R): Promise<T> {
|
||||
const state: AgentDispatchState = { followUp: null };
|
||||
dispatchStates.set(agentId, state);
|
||||
const controller = new AbortController();
|
||||
const expired = deferred<void>();
|
||||
let released = false;
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
if (dispatchStates.get(agentId) === state) dispatchStates.delete(agentId);
|
||||
const next = state.followUp;
|
||||
state.followUp = null;
|
||||
if (next) startPass(agentId, next.spec, next.request).then(next.done.resolve, next.done.reject);
|
||||
};
|
||||
const deadline = setTimeout(() => {
|
||||
logger.warn({ agentId, deadlineMs: AGENT_DISPATCH_PASS_DEADLINE_MS },
|
||||
"queued-run dispatch pass passed its deadline; it stops claiming and the next pass may start");
|
||||
controller.abort();
|
||||
release();
|
||||
expired.resolve();
|
||||
}, AGENT_DISPATCH_PASS_DEADLINE_MS);
|
||||
deadline.unref?.();
|
||||
const pass = dispatchingAgent.run(agentId, () => spec.run(request, controller.signal)).then(
|
||||
async ({ value, settle }) => {
|
||||
clearTimeout(deadline);
|
||||
release();
|
||||
if (settle) await outsideAgentDispatch(settle);
|
||||
return value;
|
||||
},
|
||||
(err: unknown) => {
|
||||
clearTimeout(deadline);
|
||||
release();
|
||||
throw err;
|
||||
},
|
||||
);
|
||||
startLocksByAgent.set(agentId, { promise: marker, startedAtMs: Date.now() });
|
||||
try {
|
||||
return await run;
|
||||
} finally {
|
||||
if (startLocksByAgent.get(agentId)?.promise === marker) {
|
||||
startLocksByAgent.delete(agentId);
|
||||
}
|
||||
}
|
||||
spec.track?.(Promise.race([pass.then(() => undefined, () => undefined), expired.promise]));
|
||||
return pass;
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche
|
||||
import { deferredWorkIntentWait } from "../work-intent-outbox.js";
|
||||
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
|
||||
import { createHeartbeatTimerPolicy } from "./timer-policy.js";
|
||||
import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged } from "./queued-run-dispatch-order.js";
|
||||
import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js";
|
||||
import { queuedRunDispatchFacts } from "./queued-run-line.js";
|
||||
import { notifyExecutionRunWork } from "../execution-control-notifications.js";
|
||||
import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
@@ -188,7 +188,8 @@ import {
|
||||
} from "../execution-workspace-policy.js";
|
||||
import { readContinuationAttempt } from "../recovery/index.js";
|
||||
import { allowsDependencyBlockedWake, extractWakeCommentIds, wakeChatConversationsConflict } from "../../modules/run-dispatch/index.js";
|
||||
import { withAgentStartLock } from "../agent-start-lock.js";
|
||||
import { dispatchForAgent, outsideAgentDispatch } from "../agent-start-lock.js";
|
||||
import { AgentRunSlotsFull, assertAgentRunSlotFree } from "./run-slot-admission.js";
|
||||
import {
|
||||
evaluateAgentInvokability,
|
||||
shouldCancelRunsForNonInvokableAgent,
|
||||
@@ -214,6 +215,9 @@ import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCom
|
||||
import { crashPoint } from "@paperclipai/shared/crash-points";
|
||||
|
||||
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
|
||||
type QueuedRunCandidate = Pick<HeartbeatRun, "id" | "companyId" | "agentId" | "status" | "invocationSource" | "createdAt"
|
||||
| "startedAt" | "wakeupRequestId" | "retryOfRunId" | "scheduledRetryAt" | "scheduledRetryAttempt" | "contextSnapshot">;
|
||||
type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean };
|
||||
type HeartbeatRunState = ReturnType<typeof createHeartbeatRunState>;
|
||||
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
|
||||
|
||||
@@ -375,6 +379,9 @@ export function shouldQueueFollowupForRunningIssueWake(input: {
|
||||
|
||||
/** Agent Chat outbox receipts. The outbox, not a skipped receipt, owns a capped message. */
|
||||
const CONVERSATION_COMMENT_KEY_PREFIX = "conversation-comment:";
|
||||
const QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS = 24;
|
||||
const QUEUED_RUN_CLAIM_BACKOFF_BASE_MS = 5_000;
|
||||
const QUEUED_RUN_CLAIM_BACKOFF_MAX_MS = 60_000;
|
||||
function dailyLimitConversationRefusal(cap: { reason: string; observed: number; limit: number }) {
|
||||
// No receipt is written, so the person's message stays owed in the outbox
|
||||
// until the limit resets (agent-conversations.ts conversationDeliveryHold).
|
||||
@@ -597,7 +604,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
|
||||
async function listQueuedRunDependencyReadiness(
|
||||
companyId: string,
|
||||
queuedRuns: Array<typeof heartbeatRuns.$inferSelect>,
|
||||
queuedRuns: Array<Pick<HeartbeatRun, "contextSnapshot">>,
|
||||
) {
|
||||
const issueIds = [
|
||||
...new Set(
|
||||
@@ -960,6 +967,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
companyAgents?: AgentOrgRow[],
|
||||
deferredPostCommitEffects?: WakeQueuePostCommitEffect[],
|
||||
deferredCancellations?: Array<() => Promise<void>>,
|
||||
slots?: { full: boolean },
|
||||
) {
|
||||
if (run.status !== "queued") return run;
|
||||
// Every start path ends here (dispatch passes, executeRun of a queued run,
|
||||
@@ -1583,7 +1591,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// envelope. Preserve their established claim path; only an
|
||||
// explicitly bound queued-message envelope is subject to the
|
||||
// live-comment discard gate below.
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId));
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
|
||||
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
|
||||
});
|
||||
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId });
|
||||
const [claimedRun] = await tx
|
||||
.update(heartbeatRuns)
|
||||
@@ -1679,7 +1690,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
};
|
||||
}
|
||||
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId));
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
|
||||
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
|
||||
});
|
||||
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId });
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
@@ -1723,6 +1737,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
.catch((error) => {
|
||||
if (isExternalChatWaitAuthorizationContention(error) || error instanceof RunAdmissionClosedDuringClaim)
|
||||
return { kind: "stale" as const, run: null };
|
||||
if (error instanceof AgentRunSlotsFull) {
|
||||
if (slots) slots.full = true;
|
||||
return { kind: "stale" as const, run: null };
|
||||
}
|
||||
throw error;
|
||||
})
|
||||
: null;
|
||||
@@ -1795,7 +1813,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId),
|
||||
)).for("update");
|
||||
if (!lockedRun || lockedRun.status !== "queued") return null;
|
||||
await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, () => assertNoExternalOverlap(claimTx, run.companyId, run.agentId));
|
||||
await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, async () => {
|
||||
await assertNoExternalOverlap(claimTx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(claimTx, run.companyId, run.agentId);
|
||||
});
|
||||
await notifyChatRunMilestones(claimTx, { companyId: run.companyId, runId: run.id, issueId: issueId ?? undefined });
|
||||
const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and(
|
||||
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"),
|
||||
@@ -1806,6 +1827,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
});
|
||||
}).catch((error: unknown) => {
|
||||
if (error instanceof RunAdmissionClosedDuringClaim) return null;
|
||||
if (error instanceof AgentRunSlotsFull) {
|
||||
if (slots) slots.full = true;
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
if (!claimed) return null;
|
||||
@@ -2252,7 +2277,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
|
||||
const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))];
|
||||
for (const agentId of agentIds) {
|
||||
await startNextQueuedRunForAgent(agentId, true);
|
||||
await startNextQueuedRunForAgent(agentId, true, true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2336,7 +2361,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
|
||||
const declinedAgedClaimAt = new Map<string, number>();
|
||||
|
||||
function noteDeclinedAgedClaim(run: typeof heartbeatRuns.$inferSelect, reason: string) {
|
||||
function noteDeclinedAgedClaim(run: QueuedRunCandidate, reason: string) {
|
||||
const now = Date.now();
|
||||
if (!hasQueuedRunAged(run, new Date(now))) return;
|
||||
declinedAgedClaimAt.set(run.id, now);
|
||||
@@ -2345,6 +2370,25 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
"long-queued heartbeat run was offered a free slot but could not be claimed; it stays queued and the next oldest run takes its turn");
|
||||
}
|
||||
|
||||
const claimBackoffs = new Map<string, { agentId: string; attempts: number; until: number }>();
|
||||
|
||||
function claimBackedOff(runId: string, now: Date) {
|
||||
return (claimBackoffs.get(runId)?.until ?? 0) > now.getTime();
|
||||
}
|
||||
|
||||
function backOffClaim(run: QueuedRunCandidate) {
|
||||
const attempts = (claimBackoffs.get(run.id)?.attempts ?? 0) + 1;
|
||||
const delayMs = Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS);
|
||||
claimBackoffs.set(run.id, { agentId: run.agentId, attempts, until: Date.now() + delayMs });
|
||||
}
|
||||
|
||||
function forgetSettledClaimBackoffs(agentId: string, queuedRuns: QueuedRunCandidate[]) {
|
||||
const queued = new Set(queuedRuns.map(run => run.id));
|
||||
for (const [runId, backoff] of claimBackoffs) {
|
||||
if (backoff.agentId === agentId && !queued.has(runId)) claimBackoffs.delete(runId);
|
||||
}
|
||||
}
|
||||
|
||||
function pruneDeclinedAgedClaims(now: Date) {
|
||||
for (const [id, at] of declinedAgedClaimAt) {
|
||||
if (now.getTime() - at >= QUEUED_RUN_AGING_MS) declinedAgedClaimAt.delete(id);
|
||||
@@ -2353,36 +2397,78 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
|
||||
async function queuedRunDispatcherForAgent(
|
||||
agent: typeof agents.$inferSelect,
|
||||
queuedRuns: Array<typeof heartbeatRuns.$inferSelect>,
|
||||
queuedRuns: QueuedRunCandidate[],
|
||||
dependencyReadiness: Map<string, { isDependencyReady: boolean }>,
|
||||
issueById: Map<string, { status: string; priority: string | null }>,
|
||||
) {
|
||||
const now = new Date();
|
||||
pruneDeclinedAgedClaims(now);
|
||||
const { facts, dispatchInput } = await queuedRunDispatchFacts(db, {
|
||||
agent, queuedRuns, dependencyReadiness, issueById, now, shareDeferred: runId => declinedAgedClaimAt.has(runId) });
|
||||
agent, queuedRuns, dependencyReadiness, issueById, now,
|
||||
shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now) });
|
||||
const dispatcher = createQueuedRunDispatcher(facts, dispatchInput);
|
||||
const byRun = new Map(facts.map(entry => [entry.run, entry]));
|
||||
return {
|
||||
next: () => dispatcher.next()?.run ?? null,
|
||||
claimed: (run: typeof heartbeatRuns.$inferSelect) => { const entry = byRun.get(run); if (entry) dispatcher.claimed(entry); },
|
||||
claimed: (run: QueuedRunCandidate) => { const entry = byRun.get(run); if (entry) dispatcher.claimed(entry); },
|
||||
personRun: (run: QueuedRunCandidate) => {
|
||||
const entry = byRun.get(run);
|
||||
return Boolean(entry && queuedRunDispatchClass(entry, now) === "person");
|
||||
},
|
||||
personOrigin: (run: QueuedRunCandidate) => byRun.get(run)?.requestedByUser ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
async function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false) {
|
||||
// Null unless this server has not read the run-start hand-over recently.
|
||||
const refreshing = refreshRunAdmission();
|
||||
if (refreshing) await refreshing;
|
||||
if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return [];
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
// Cancelled after the start lock is released: cancelRunInternal promotes the
|
||||
// agent's next queued run, which takes this same lock.
|
||||
function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false, retryRefused = false) {
|
||||
return dispatchForAgent<QueuedDispatchRequest, Array<typeof heartbeatRuns.$inferSelect>>(agentId, {
|
||||
request: { attachSetupRequest, retryRefused },
|
||||
idle: [],
|
||||
merge: (current, next) => ({
|
||||
attachSetupRequest: current.attachSetupRequest || next.attachSetupRequest,
|
||||
retryRefused: current.retryRefused || next.retryRefused,
|
||||
}),
|
||||
run: (request, signal) => runQueuedDispatchPass(agentId, request, signal),
|
||||
track: (pass) => {
|
||||
const tracked = pass.then(() => undefined);
|
||||
activeRunExecutionPromises.add(tracked);
|
||||
void tracked.finally(() => activeRunExecutionPromises.delete(tracked));
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused }: QueuedDispatchRequest, signal: AbortSignal) {
|
||||
// Cancelled after the pass ends: cancelRunInternal promotes the agent's
|
||||
// next queued run, which asks for another pass of this agent.
|
||||
const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = [];
|
||||
const deferredPostCommitEffects: WakeQueuePostCommitEffect[] = [];
|
||||
const deferredCancellations: Array<() => Promise<void>> = [];
|
||||
let cancellationReason: string | undefined;
|
||||
const settle = async () => {
|
||||
// Dispatch promoted inputs after the pass has ended.
|
||||
try {
|
||||
await applyWakeQueuePostCommitEffects(deferredPostCommitEffects);
|
||||
} finally {
|
||||
if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason);
|
||||
await cancelRejectedQueuedRuns(rejectedClaims);
|
||||
for (const cancel of deferredCancellations) {
|
||||
await cancel().catch((err) => logger.warn({ err, agentId },
|
||||
"queued heartbeat run refused by a pause hold could not be cancelled; the next claim retries it"));
|
||||
}
|
||||
}
|
||||
};
|
||||
try {
|
||||
return { value: await claimQueuedRunsForAgent(), settle };
|
||||
} catch (err) {
|
||||
await settle();
|
||||
throw err;
|
||||
}
|
||||
|
||||
return withAgentStartLock(agentId, async () => {
|
||||
async function claimQueuedRunsForAgent(): Promise<Array<typeof heartbeatRuns.$inferSelect>> {
|
||||
// Null unless this server has not read the run-start hand-over recently.
|
||||
const refreshing = refreshRunAdmission();
|
||||
if (refreshing) await refreshing;
|
||||
if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return [];
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const agent = await getAgent(agentId);
|
||||
if (!agent) return [];
|
||||
const invokability = await getAgentInvokability(agent);
|
||||
@@ -2395,64 +2481,44 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}
|
||||
const policy = parseHeartbeatPolicy(agent);
|
||||
const runningCount = await countRunningRunsForAgent(agentId);
|
||||
const availableSlots = Math.max(
|
||||
0,
|
||||
policy.maxConcurrentRuns - runningCount,
|
||||
);
|
||||
const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount);
|
||||
if (availableSlots <= 0) return [];
|
||||
|
||||
const queuedRuns = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.agentId, agentId),
|
||||
eq(heartbeatRuns.status, "queued"),
|
||||
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
|
||||
),
|
||||
)
|
||||
.orderBy(asc(heartbeatRuns.createdAt));
|
||||
const queuedRuns = await listQueuedRunCandidates(agentId, cutoff);
|
||||
forgetSettledClaimBackoffs(agentId, queuedRuns);
|
||||
if (queuedRuns.length === 0) return [];
|
||||
if (queuedRuns.every(run => !run.startedAt) && !await canStartNewExternalBotWork(agent)) return [];
|
||||
|
||||
const dependencyReadiness = await listQueuedRunDependencyReadiness(
|
||||
agent.companyId,
|
||||
queuedRuns,
|
||||
);
|
||||
const queuedIssueIds = [
|
||||
...new Set(
|
||||
queuedRuns
|
||||
.map((run) =>
|
||||
readNonEmptyString(parseObject(run.contextSnapshot).issueId),
|
||||
)
|
||||
.filter((issueId): issueId is string => Boolean(issueId)),
|
||||
),
|
||||
];
|
||||
const issueRows = await db
|
||||
.select({
|
||||
id: issues.id,
|
||||
status: issues.status,
|
||||
priority: issues.priority,
|
||||
})
|
||||
.from(issues)
|
||||
.where(
|
||||
queuedIssueIds.length > 0
|
||||
? and(
|
||||
eq(issues.companyId, agent.companyId),
|
||||
inArray(issues.id, queuedIssueIds),
|
||||
)
|
||||
: sql`false`,
|
||||
);
|
||||
const dependencyReadiness = await listQueuedRunDependencyReadiness(agent.companyId, queuedRuns);
|
||||
const queuedIssueIds = [...new Set(queuedRuns.flatMap((run) => {
|
||||
const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
|
||||
return issueId ? [issueId] : [];
|
||||
}))];
|
||||
const issueRows = queuedIssueIds.length > 0
|
||||
? await db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues)
|
||||
.where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds)))
|
||||
: [];
|
||||
const issueById = new Map(issueRows.map((row) => [row.id, row]));
|
||||
const companyAgents = await listCompanyAgentOrgRows(agent.companyId);
|
||||
const dispatcher = await queuedRunDispatcherForAgent(agent, queuedRuns, dependencyReadiness, issueById);
|
||||
|
||||
const claimedRuns: Array<typeof heartbeatRuns.$inferSelect> = [];
|
||||
for (let queuedRun = dispatcher.next(); queuedRun; queuedRun = dispatcher.next()) {
|
||||
if (claimedRuns.length >= availableSlots || isRunAdmissionClosed()) break;
|
||||
const slots = { full: false };
|
||||
let claimAttempts = 0;
|
||||
for (let candidate = dispatcher.next(); candidate; candidate = dispatcher.next()) {
|
||||
if (claimedRuns.length >= availableSlots || slots.full || isRunAdmissionClosed() || signal.aborted) break;
|
||||
if (!retryRefused && !dispatcher.personRun(candidate) && claimBackedOff(candidate.id, new Date())) continue;
|
||||
if (claimAttempts >= QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS) {
|
||||
void startNextQueuedRunForAgent(agentId);
|
||||
break;
|
||||
}
|
||||
claimAttempts += 1;
|
||||
const [queuedRun] = await db.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.status, "queued")));
|
||||
if (!queuedRun) continue;
|
||||
let claimed: typeof heartbeatRuns.$inferSelect | null;
|
||||
try {
|
||||
claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects, deferredCancellations);
|
||||
claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects, deferredCancellations, slots);
|
||||
} catch (err) {
|
||||
if (isPermanentClaimRejection(err)) {
|
||||
rejectedClaims.push({ run: queuedRun, err });
|
||||
@@ -2463,54 +2529,75 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
{ err, runId: queuedRun.id, agentId: queuedRun.agentId, companyId: queuedRun.companyId },
|
||||
"queued heartbeat run claim was rejected; leaving it queued for the next recovery pass",
|
||||
);
|
||||
noteDeclinedAgedClaim(queuedRun, err.message);
|
||||
noteDeclinedAgedClaim(candidate, err.message);
|
||||
backOffClaim(candidate);
|
||||
continue;
|
||||
}
|
||||
if (!claimed) {
|
||||
noteDeclinedAgedClaim(queuedRun, "claim_declined");
|
||||
if (slots.full) break;
|
||||
noteDeclinedAgedClaim(candidate, "claim_declined");
|
||||
backOffClaim(candidate);
|
||||
continue;
|
||||
}
|
||||
claimedRuns.push(claimed);
|
||||
dispatcher.claimed(queuedRun);
|
||||
declinedAgedClaimAt.delete(queuedRun.id);
|
||||
dispatcher.claimed(candidate);
|
||||
declinedAgedClaimAt.delete(candidate.id);
|
||||
claimBackoffs.delete(candidate.id);
|
||||
// Start each run as soon as it is claimed. The claim committed a legacy
|
||||
// controller lease that only the run's own execution renews; waiting
|
||||
// for the rest of a slow batch to be claimed let that lease lapse.
|
||||
startClaimedRun(claimed);
|
||||
}
|
||||
return claimedRuns;
|
||||
}
|
||||
|
||||
function startClaimedRun(claimedRun: typeof heartbeatRuns.$inferSelect) {
|
||||
const execution = executeRun(claimedRun.id).catch((err) => {
|
||||
logger.error(
|
||||
{ err, runId: claimedRun.id },
|
||||
"queued heartbeat execution failed",
|
||||
);
|
||||
});
|
||||
// Register the in-flight execution so drainActiveRunExecutions() can await
|
||||
// it. executeRun resolves only after its finally block finishes flushing
|
||||
// run rows/events, so awaiting this promise guarantees the run's writes
|
||||
// have landed before a caller (e.g. a test's afterEach) mutates the DB.
|
||||
activeRunExecutionPromises.add(execution);
|
||||
void execution.finally(() => {
|
||||
// drainActiveRunExecutions loops on activeRunExecutionPromises.size,
|
||||
// so an entry that never clears here would hang it forever.
|
||||
activeRunExecutionPromises.delete(execution);
|
||||
});
|
||||
}
|
||||
}).finally(async () => {
|
||||
// Dispatch promoted inputs after the agent start lock is released.
|
||||
try {
|
||||
await applyWakeQueuePostCommitEffects(deferredPostCommitEffects);
|
||||
} finally {
|
||||
if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason);
|
||||
await cancelRejectedQueuedRuns(rejectedClaims);
|
||||
for (const cancel of deferredCancellations) {
|
||||
await cancel().catch((err) => logger.warn({ err, agentId },
|
||||
"queued heartbeat run refused by a pause hold could not be cancelled; the next claim retries it"));
|
||||
}
|
||||
}
|
||||
});
|
||||
function startClaimedRun(claimedRun: typeof heartbeatRuns.$inferSelect) {
|
||||
const execution = outsideAgentDispatch(() => executeRun(claimedRun.id)).catch((err) => {
|
||||
logger.error(
|
||||
{ err, runId: claimedRun.id },
|
||||
"queued heartbeat execution failed",
|
||||
);
|
||||
});
|
||||
// Register the in-flight execution so drainActiveRunExecutions() can await
|
||||
// it. executeRun resolves only after its finally block finishes flushing
|
||||
// run rows/events, so awaiting this promise guarantees the run's writes
|
||||
// have landed before a caller (e.g. a test's afterEach) mutates the DB.
|
||||
activeRunExecutionPromises.add(execution);
|
||||
void execution.finally(() => {
|
||||
// drainActiveRunExecutions loops on activeRunExecutionPromises.size,
|
||||
// so an entry that never clears here would hang it forever.
|
||||
activeRunExecutionPromises.delete(execution);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function listQueuedRunCandidates(agentId: string, cutoff: Date | null): Promise<QueuedRunCandidate[]> {
|
||||
const contextText = (key: string) => sql<string | null>`case when jsonb_typeof(${heartbeatRuns.contextSnapshot}->${key}) = 'string'
|
||||
then ${heartbeatRuns.contextSnapshot}->>${key} end`;
|
||||
const rows = await db.select({
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
status: heartbeatRuns.status,
|
||||
invocationSource: heartbeatRuns.invocationSource,
|
||||
createdAt: heartbeatRuns.createdAt,
|
||||
startedAt: heartbeatRuns.startedAt,
|
||||
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
||||
retryOfRunId: heartbeatRuns.retryOfRunId,
|
||||
scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
|
||||
scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt,
|
||||
issueId: contextText("issueId"),
|
||||
wakeReason: contextText("wakeReason"),
|
||||
previousRunId: contextText("previousRunId"),
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.agentId, agentId),
|
||||
eq(heartbeatRuns.status, "queued"),
|
||||
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
|
||||
)).orderBy(asc(heartbeatRuns.createdAt));
|
||||
return rows.map(({ issueId, wakeReason, previousRunId, ...run }) => ({
|
||||
...run,
|
||||
contextSnapshot: Object.fromEntries(Object.entries({ issueId, wakeReason, previousRunId }).filter(([, value]) => value !== null)),
|
||||
}));
|
||||
}
|
||||
|
||||
// Public wakeup entry point. Callers dispatch it fire-and-forget, so register
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { maxConcurrentRunsSql } from "./timer-policy.js";
|
||||
|
||||
export class AgentRunSlotsFull extends Error {
|
||||
constructor(readonly agentId: string, readonly running: number, readonly limit: number) {
|
||||
super("The agent already runs as many runs at once as it allows.");
|
||||
this.name = "AgentRunSlotsFull";
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertAgentRunSlotFree(tx: Pick<Db, "select">, companyId: string, agentId: string) {
|
||||
const [slots] = await tx.select({
|
||||
limit: maxConcurrentRunsSql(),
|
||||
running: sql<number>`(select count(*)::int from ${heartbeatRuns} where ${heartbeatRuns.companyId} = ${companyId}::uuid and ${heartbeatRuns.agentId} = ${agentId}::uuid and ${heartbeatRuns.status} = 'running')`,
|
||||
}).from(agents).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
|
||||
if (!slots) return;
|
||||
const running = Number(slots.running);
|
||||
const limit = Number(slots.limit);
|
||||
if (running >= limit) throw new AgentRunSlotsFull(agentId, running, limit);
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { withExternalAdmissionGuard, assertNoExternalOverlap } from "../../modul
|
||||
import { agentWakeupRequests, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import type { PgUpdateSetSource } from "drizzle-orm/pg-core";
|
||||
import { assertAgentRunSlotFree } from "../heartbeat/run-slot-admission.js";
|
||||
import { claimNativeReviewExecutionLock, readNativeReviewAssignmentContext } from "./native-review-participant.js";
|
||||
|
||||
/** The run, wake, and issue lock form one reviewer admission claim. */
|
||||
@@ -35,7 +36,10 @@ export async function claimQueuedNativeReviewRun(db: Db, input: {
|
||||
)).for("update");
|
||||
if (!run || run.status !== "queued" || run.wakeupRequestId !== input.run.wakeupRequestId) return null;
|
||||
if ((run.contextSnapshot as Record<string, unknown> | null)?.issueId !== issueId) return null;
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId));
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
|
||||
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
|
||||
});
|
||||
const locked = await claimNativeReviewExecutionLock(tx as unknown as Db, {
|
||||
companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id,
|
||||
contextSnapshot: context, agentNameKey: input.agentNameKey, claimedAt: input.claimedAt,
|
||||
|
||||
@@ -48,7 +48,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
const experimental = await instanceSettingsService(db).getExperimental();
|
||||
let stoppedWorker: Awaited<ReturnType<typeof stoppedWorkspaceWorker>> | undefined;
|
||||
let storage: { mockRestore(): void } | undefined;
|
||||
const starts = vi.spyOn(agentStarts, "withAgentStartLock");
|
||||
const starts = vi.spyOn(agentStarts, "dispatchForAgent");
|
||||
const retryResumes: Promise<void>[] = [];
|
||||
let resumeSpy: { mockRestore(): void } | undefined;
|
||||
let ownedHeartbeat: ReturnType<typeof heartbeatService> | undefined;
|
||||
@@ -178,7 +178,8 @@ describe("automatic retained workspace recovery", () => {
|
||||
});
|
||||
const app = boardRecoveryApp(undefined, heartbeat);
|
||||
const gate = new Promise<void>(resolve => { releaseStart = resolve; });
|
||||
heldStart = agentStarts.withAgentStartLock(source.agentId, () => gate);
|
||||
heldStart = agentStarts.dispatchForAgent(source.agentId, { request: null, idle: undefined, merge: () => null,
|
||||
run: async () => ({ value: await gate }) });
|
||||
const receipt = mode === "automatic_lost" ? null : (await request(app).post(`/api/issues/${f.issueId}/retry`)
|
||||
.send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body;
|
||||
if (mode === "automatic_lost") {
|
||||
|
||||
Reference in new issue
Block a user