mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(heartbeat): let successors start after their predecessor and stop chaining passes on refused runs
Back off a refused claim only when the run has waited past the aging limit, end a backoff when the run's task changes or a run of that task ends, and count a successor's slot without the predecessor it replaces. A pass that reaches its claim budget hands the runs it tried to one follow-up pass, so refused person runs cannot hold back runnable work or chain passes. A trigger after a pass's deadline also ends that pass, measured on a monotonic clock from when it started. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
68e64a2de9
commit
3d48c4bed2
7 files changed
+155
-37
No files matched your search
@@ -234,4 +234,56 @@ suite("queued-run dispatch under a trigger burst", () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
async function passesSettle() {
|
||||
let last = -1;
|
||||
await vi.waitFor(() => {
|
||||
const settled = passes.started === last && !passes.active;
|
||||
last = passes.started;
|
||||
expect(settled).toBe(true);
|
||||
}, { timeout: 30_000, interval: 1_000 });
|
||||
return passes.started;
|
||||
}
|
||||
|
||||
it("stops chaining passes once every refused run was tried", async () => {
|
||||
passes.started = 0; passes.active = 0; passes.peak = 0;
|
||||
const agentId = await seedAgent();
|
||||
const otherTask = await task(agentId);
|
||||
for (let index = 0; index < 60; index += 1) {
|
||||
await queuedContinuation(agentId, { minutesAgo: 180 - index, wakeReason: "issue_continuation_needed", sourceIssueId: otherTask.id });
|
||||
}
|
||||
const heartbeat = heartbeatService(db);
|
||||
try {
|
||||
await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
|
||||
expect(await passesSettle()).toBe(3);
|
||||
expect(passes.peak).toBe(1);
|
||||
} finally {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("starts runnable work queued behind more refused person runs than one pass tries", async () => {
|
||||
held.reset();
|
||||
passes.started = 0; passes.active = 0; passes.peak = 0;
|
||||
const agentId = await seedAgent();
|
||||
const otherTask = await task(agentId);
|
||||
for (let index = 0; index < 30; index += 1) {
|
||||
await queuedContinuation(agentId, { minutesAgo: 90 - index, wakeReason: "task_retry", sourceIssueId: otherTask.id });
|
||||
}
|
||||
const runnable: string[] = [];
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
runnable.push((await queuedContinuation(agentId, { minutesAgo: 30 - index, wakeReason: "issue_assigned" })).id);
|
||||
}
|
||||
const heartbeat = heartbeatService(db);
|
||||
try {
|
||||
await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
|
||||
expect(await passesSettle()).toBeLessThanOrEqual(3);
|
||||
const started = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
|
||||
.where(sql`${heartbeatRuns.id} in (${sql.join(runnable.map(id => sql`${id}::uuid`), sql`, `)}) and ${heartbeatRuns.startedAt} is not null`);
|
||||
expect(started).toHaveLength(runnable.length);
|
||||
} finally {
|
||||
held.release();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
}
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -95,6 +95,21 @@ describe("agent dispatch passes", () => {
|
||||
expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(3);
|
||||
});
|
||||
|
||||
it("lets a trigger stop a hung pass by the pass's start time when its deadline timer has not run", async () => {
|
||||
vi.useFakeTimers({ toFake: ["performance"] });
|
||||
const agentId = randomUUID();
|
||||
const { stats, spec } = meter();
|
||||
let hungSignal: AbortSignal | null = null;
|
||||
void dispatchForAgent(agentId, spec(false, (signal) => {
|
||||
hungSignal = signal;
|
||||
return new Promise<void>(() => undefined);
|
||||
}));
|
||||
vi.advanceTimersByTime(AGENT_DISPATCH_PASS_DEADLINE_MS);
|
||||
expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(2);
|
||||
expect(hungSignal!.aborted).toBe(true);
|
||||
expect(stats.passes).toBe(2);
|
||||
});
|
||||
|
||||
it("does not wait on itself when a pass triggers its own agent", async () => {
|
||||
const agentId = randomUUID();
|
||||
const { stats, spec } = meter();
|
||||
|
||||
@@ -13,7 +13,7 @@ export type AgentDispatchPass<R, T> = {
|
||||
|
||||
type Deferred<T> = { promise: Promise<T>; resolve(value: T): void; reject(err: unknown): void };
|
||||
type FollowUp = { request: unknown; spec: AgentDispatchPass<unknown, unknown>; done: Deferred<unknown> };
|
||||
type AgentDispatchState = { followUp: FollowUp | null };
|
||||
type AgentDispatchState = { followUp: FollowUp | null; startedAtMs: number; expire(): void };
|
||||
|
||||
const dispatchStates = new Map<string, AgentDispatchState>();
|
||||
const dispatchingAgent = new AsyncLocalStorage<string>();
|
||||
@@ -38,7 +38,11 @@ export function outsideAgentDispatch<T>(fn: () => T): T {
|
||||
}
|
||||
|
||||
export function dispatchForAgent<R, T>(agentId: string, spec: AgentDispatchPass<R, T>): Promise<T> {
|
||||
const state = dispatchStates.get(agentId);
|
||||
let state = dispatchStates.get(agentId);
|
||||
if (state && performance.now() - state.startedAtMs >= AGENT_DISPATCH_PASS_DEADLINE_MS) {
|
||||
state.expire();
|
||||
state = dispatchStates.get(agentId);
|
||||
}
|
||||
if (!state) return startPass(agentId, spec, spec.request);
|
||||
if (state.followUp) {
|
||||
state.followUp.request = spec.merge(state.followUp.request as R, spec.request);
|
||||
@@ -51,10 +55,10 @@ export function dispatchForAgent<R, T>(agentId: string, spec: AgentDispatchPass<
|
||||
}
|
||||
|
||||
function startPass<R, T>(agentId: string, spec: AgentDispatchPass<R, T>, request: R): Promise<T> {
|
||||
const state: AgentDispatchState = { followUp: null };
|
||||
dispatchStates.set(agentId, state);
|
||||
const controller = new AbortController();
|
||||
const expired = deferred<void>();
|
||||
const state: AgentDispatchState = { followUp: null, startedAtMs: performance.now(), expire: () => expire() };
|
||||
dispatchStates.set(agentId, state);
|
||||
let released = false;
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
@@ -64,13 +68,16 @@ function startPass<R, T>(agentId: string, spec: AgentDispatchPass<R, T>, request
|
||||
state.followUp = null;
|
||||
if (next) startPass(agentId, next.spec, next.request).then(next.done.resolve, next.done.reject);
|
||||
};
|
||||
const deadline = setTimeout(() => {
|
||||
const expire = () => {
|
||||
if (released) return;
|
||||
clearTimeout(deadline);
|
||||
logger.warn({ agentId, deadlineMs: AGENT_DISPATCH_PASS_DEADLINE_MS },
|
||||
"queued-run dispatch pass passed its deadline; it stops claiming and the next pass may start");
|
||||
controller.abort();
|
||||
release();
|
||||
expired.resolve();
|
||||
}, AGENT_DISPATCH_PASS_DEADLINE_MS);
|
||||
};
|
||||
const deadline = setTimeout(expire, AGENT_DISPATCH_PASS_DEADLINE_MS);
|
||||
deadline.unref?.();
|
||||
const pass = dispatchingAgent.run(agentId, () => spec.run(request, controller.signal)).then(
|
||||
async ({ value, settle }) => {
|
||||
|
||||
@@ -4388,6 +4388,7 @@ export function heartbeatService(
|
||||
logger.warn({ err, runId: run.id }, "failed to deliver settled tool reviews after execution cleanup");
|
||||
});
|
||||
}
|
||||
heartbeatQueue.releaseClaimBackoffsForTask(latestRun ?? run);
|
||||
await startNextQueuedRunForAgent(run.agentId);
|
||||
if (latestRun && isHeartbeatRunTerminalStatus(latestRun.status)) {
|
||||
await notifyOtherAgentsQueuedOnSettledIssue(db, latestRun).catch(err => {
|
||||
|
||||
@@ -216,8 +216,9 @@ import { crashPoint } from "@paperclipai/shared/crash-points";
|
||||
|
||||
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
|
||||
type QueuedRunCandidate = Pick<HeartbeatRun, "id" | "companyId" | "agentId" | "status" | "invocationSource" | "createdAt"
|
||||
| "startedAt" | "wakeupRequestId" | "retryOfRunId" | "scheduledRetryAt" | "scheduledRetryAttempt" | "contextSnapshot">;
|
||||
type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean };
|
||||
| "startedAt" | "wakeupRequestId" | "retryOfRunId" | "scheduledRetryAt" | "scheduledRetryAttempt" | "scheduledRetryReason"
|
||||
| "contextSnapshot">;
|
||||
type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean; triedRunIds: ReadonlySet<string> };
|
||||
type HeartbeatRunState = ReturnType<typeof createHeartbeatRunState>;
|
||||
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
|
||||
|
||||
@@ -382,6 +383,7 @@ const CONVERSATION_COMMENT_KEY_PREFIX = "conversation-comment:";
|
||||
const QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS = 24;
|
||||
const QUEUED_RUN_CLAIM_BACKOFF_BASE_MS = 5_000;
|
||||
const QUEUED_RUN_CLAIM_BACKOFF_MAX_MS = 60_000;
|
||||
const QUEUED_PERSON_RUN_CLAIM_BACKOFF_MS = 2_000;
|
||||
function dailyLimitConversationRefusal(cap: { reason: string; observed: number; limit: number }) {
|
||||
// No receipt is written, so the person's message stays owed in the outbox
|
||||
// until the limit resets (agent-conversations.ts conversationDeliveryHold).
|
||||
@@ -1593,7 +1595,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// live-comment discard gate below.
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
|
||||
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run);
|
||||
});
|
||||
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId });
|
||||
const [claimedRun] = await tx
|
||||
@@ -1692,7 +1694,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
|
||||
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run);
|
||||
});
|
||||
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId });
|
||||
await tx
|
||||
@@ -1815,7 +1817,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (!lockedRun || lockedRun.status !== "queued") return null;
|
||||
await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, async () => {
|
||||
await assertNoExternalOverlap(claimTx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(claimTx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(claimTx, run);
|
||||
});
|
||||
await notifyChatRunMilestones(claimTx, { companyId: run.companyId, runId: run.id, issueId: issueId ?? undefined });
|
||||
const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and(
|
||||
@@ -2370,16 +2372,36 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
"long-queued heartbeat run was offered a free slot but could not be claimed; it stays queued and the next oldest run takes its turn");
|
||||
}
|
||||
|
||||
const claimBackoffs = new Map<string, { agentId: string; attempts: number; until: number }>();
|
||||
const claimBackoffs = new Map<string, { agentId: string; issueId: string | null; attempts: number; until: number;
|
||||
taskStamp: string | null }>();
|
||||
|
||||
function claimBackedOff(runId: string, now: Date) {
|
||||
return (claimBackoffs.get(runId)?.until ?? 0) > now.getTime();
|
||||
function claimBackedOff(runId: string, now: Date, taskStamp: string | null) {
|
||||
const backoff = claimBackoffs.get(runId);
|
||||
return Boolean(backoff && backoff.until > now.getTime() && backoff.taskStamp === taskStamp);
|
||||
}
|
||||
|
||||
function backOffClaim(run: QueuedRunCandidate) {
|
||||
function backOffClaim(run: QueuedRunCandidate, personRun: boolean, taskStamp: string | null) {
|
||||
if (!hasQueuedRunAged(run, new Date())) return;
|
||||
const attempts = (claimBackoffs.get(run.id)?.attempts ?? 0) + 1;
|
||||
const delayMs = Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS);
|
||||
claimBackoffs.set(run.id, { agentId: run.agentId, attempts, until: Date.now() + delayMs });
|
||||
const delayMs = personRun ? QUEUED_PERSON_RUN_CLAIM_BACKOFF_MS
|
||||
: Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS);
|
||||
claimBackoffs.set(run.id, { agentId: run.agentId, issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),
|
||||
attempts, until: Date.now() + delayMs, taskStamp });
|
||||
}
|
||||
|
||||
function releaseClaimBackoffsForTask(run: Pick<HeartbeatRun, "issueId" | "contextSnapshot">) {
|
||||
const issueId = run.issueId ?? readNonEmptyString(parseObject(run.contextSnapshot).issueId);
|
||||
if (!issueId) return;
|
||||
for (const [runId, backoff] of claimBackoffs) {
|
||||
if (backoff.issueId === issueId) claimBackoffs.delete(runId);
|
||||
}
|
||||
}
|
||||
|
||||
function queuedRunTaskStamp(run: QueuedRunCandidate,
|
||||
issueById: Map<string, { status: string; updatedAt: Date; executionRunId: string | null }>) {
|
||||
const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
|
||||
const issue = issueId ? issueById.get(issueId) : null;
|
||||
return issue ? `${issue.status}:${issue.updatedAt.getTime()}:${issue.executionRunId ?? ""}` : null;
|
||||
}
|
||||
|
||||
function forgetSettledClaimBackoffs(agentId: string, queuedRuns: QueuedRunCandidate[]) {
|
||||
@@ -2399,13 +2421,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
agent: typeof agents.$inferSelect,
|
||||
queuedRuns: QueuedRunCandidate[],
|
||||
dependencyReadiness: Map<string, { isDependencyReady: boolean }>,
|
||||
issueById: Map<string, { status: string; priority: string | null }>,
|
||||
issueById: Map<string, { status: string; priority: string | null; updatedAt: Date; executionRunId: string | null }>,
|
||||
) {
|
||||
const now = new Date();
|
||||
pruneDeclinedAgedClaims(now);
|
||||
const runById = new Map(queuedRuns.map(run => [run.id, run]));
|
||||
const { facts, dispatchInput } = await queuedRunDispatchFacts(db, {
|
||||
agent, queuedRuns, dependencyReadiness, issueById, now,
|
||||
shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now) });
|
||||
shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now, queuedRunTaskStamp(runById.get(runId)!, issueById)) });
|
||||
const dispatcher = createQueuedRunDispatcher(facts, dispatchInput);
|
||||
const byRun = new Map(facts.map(entry => [entry.run, entry]));
|
||||
return {
|
||||
@@ -2420,12 +2443,17 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}
|
||||
|
||||
function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false, retryRefused = false) {
|
||||
return requestQueuedDispatch(agentId, { attachSetupRequest, retryRefused, triedRunIds: new Set() });
|
||||
}
|
||||
|
||||
function requestQueuedDispatch(agentId: string, request: QueuedDispatchRequest) {
|
||||
return dispatchForAgent<QueuedDispatchRequest, Array<typeof heartbeatRuns.$inferSelect>>(agentId, {
|
||||
request: { attachSetupRequest, retryRefused },
|
||||
request,
|
||||
idle: [],
|
||||
merge: (current, next) => ({
|
||||
attachSetupRequest: current.attachSetupRequest || next.attachSetupRequest,
|
||||
retryRefused: current.retryRefused || next.retryRefused,
|
||||
triedRunIds: new Set([...current.triedRunIds, ...next.triedRunIds]),
|
||||
}),
|
||||
run: (request, signal) => runQueuedDispatchPass(agentId, request, signal),
|
||||
track: (pass) => {
|
||||
@@ -2436,7 +2464,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
});
|
||||
}
|
||||
|
||||
async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused }: QueuedDispatchRequest, signal: AbortSignal) {
|
||||
async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused, triedRunIds }: QueuedDispatchRequest,
|
||||
signal: AbortSignal) {
|
||||
// Cancelled after the pass ends: cancelRunInternal promotes the agent's
|
||||
// next queued run, which asks for another pass of this agent.
|
||||
const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = [];
|
||||
@@ -2495,7 +2524,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
return issueId ? [issueId] : [];
|
||||
}))];
|
||||
const issueRows = queuedIssueIds.length > 0
|
||||
? await db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues)
|
||||
? await db.select({ id: issues.id, status: issues.status, priority: issues.priority, updatedAt: issues.updatedAt,
|
||||
executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds)))
|
||||
: [];
|
||||
const issueById = new Map(issueRows.map((row) => [row.id, row]));
|
||||
@@ -2505,14 +2535,18 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
const claimedRuns: Array<typeof heartbeatRuns.$inferSelect> = [];
|
||||
const slots = { full: false };
|
||||
let claimAttempts = 0;
|
||||
const triedThisPass: string[] = [];
|
||||
for (let candidate = dispatcher.next(); candidate; candidate = dispatcher.next()) {
|
||||
if (claimedRuns.length >= availableSlots || slots.full || isRunAdmissionClosed() || signal.aborted) break;
|
||||
if (!retryRefused && !dispatcher.personRun(candidate) && claimBackedOff(candidate.id, new Date())) continue;
|
||||
const taskStamp = queuedRunTaskStamp(candidate, issueById);
|
||||
if (triedRunIds.has(candidate.id) || (!retryRefused && claimBackedOff(candidate.id, new Date(), taskStamp))) continue;
|
||||
if (claimAttempts >= QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS) {
|
||||
void startNextQueuedRunForAgent(agentId);
|
||||
void requestQueuedDispatch(agentId, { attachSetupRequest: false, retryRefused,
|
||||
triedRunIds: new Set([...triedRunIds, ...triedThisPass]) });
|
||||
break;
|
||||
}
|
||||
claimAttempts += 1;
|
||||
triedThisPass.push(candidate.id);
|
||||
const [queuedRun] = await db.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.status, "queued")));
|
||||
if (!queuedRun) continue;
|
||||
@@ -2530,13 +2564,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
"queued heartbeat run claim was rejected; leaving it queued for the next recovery pass",
|
||||
);
|
||||
noteDeclinedAgedClaim(candidate, err.message);
|
||||
backOffClaim(candidate);
|
||||
backOffClaim(candidate, dispatcher.personRun(candidate), taskStamp);
|
||||
continue;
|
||||
}
|
||||
if (!claimed) {
|
||||
if (slots.full) break;
|
||||
noteDeclinedAgedClaim(candidate, "claim_declined");
|
||||
backOffClaim(candidate);
|
||||
backOffClaim(candidate, dispatcher.personRun(candidate), taskStamp);
|
||||
continue;
|
||||
}
|
||||
claimedRuns.push(claimed);
|
||||
@@ -2586,6 +2620,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
retryOfRunId: heartbeatRuns.retryOfRunId,
|
||||
scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
|
||||
scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt,
|
||||
scheduledRetryReason: heartbeatRuns.scheduledRetryReason,
|
||||
issueId: contextText("issueId"),
|
||||
wakeReason: contextText("wakeReason"),
|
||||
previousRunId: contextText("previousRunId"),
|
||||
@@ -5741,6 +5776,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
createNativeStatusWakeBatch,
|
||||
startNextQueuedRunForAgent,
|
||||
countRunningRunsForAgent,
|
||||
releaseClaimBackoffsForTask,
|
||||
releaseRunClaimedJustBeforeSuppression,
|
||||
claimQueuedRun,
|
||||
withChatControlRecoveryGate,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { isUuidLike } from "@paperclipai/shared";
|
||||
import { and, eq, notInArray, sql } from "drizzle-orm";
|
||||
import { maxConcurrentRunsSql } from "./timer-policy.js";
|
||||
|
||||
export class AgentRunSlotsFull extends Error {
|
||||
@@ -9,13 +10,19 @@ export class AgentRunSlotsFull extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertAgentRunSlotFree(tx: Pick<Db, "select">, companyId: string, agentId: string) {
|
||||
const [slots] = await tx.select({
|
||||
limit: maxConcurrentRunsSql(),
|
||||
running: sql<number>`(select count(*)::int from ${heartbeatRuns} where ${heartbeatRuns.companyId} = ${companyId}::uuid and ${heartbeatRuns.agentId} = ${agentId}::uuid and ${heartbeatRuns.status} = 'running')`,
|
||||
}).from(agents).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
|
||||
if (!slots) return;
|
||||
const running = Number(slots.running);
|
||||
const limit = Number(slots.limit);
|
||||
if (running >= limit) throw new AgentRunSlotsFull(agentId, running, limit);
|
||||
type ClaimingRun = Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId" | "agentId" | "retryOfRunId" | "contextSnapshot">;
|
||||
|
||||
export async function assertAgentRunSlotFree(tx: Pick<Db, "select">, run: ClaimingRun) {
|
||||
const previousRunId = (run.contextSnapshot as Record<string, unknown> | null)?.previousRunId;
|
||||
const replaced = [run.id, run.retryOfRunId, typeof previousRunId === "string" ? previousRunId : null]
|
||||
.filter((id): id is string => typeof id === "string" && isUuidLike(id));
|
||||
const [running] = await tx.select({ count: sql<number>`count(*)::int` }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), eq(heartbeatRuns.status, "running"),
|
||||
notInArray(heartbeatRuns.id, replaced)));
|
||||
const [agent] = await tx.select({ limit: maxConcurrentRunsSql() }).from(agents)
|
||||
.where(and(eq(agents.id, run.agentId), eq(agents.companyId, run.companyId)));
|
||||
if (!agent) return;
|
||||
const count = Number(running?.count ?? 0);
|
||||
const limit = Number(agent.limit);
|
||||
if (count >= limit) throw new AgentRunSlotsFull(run.agentId, count, limit);
|
||||
}
|
||||
@@ -38,7 +38,7 @@ export async function claimQueuedNativeReviewRun(db: Db, input: {
|
||||
if ((run.contextSnapshot as Record<string, unknown> | null)?.issueId !== issueId) return null;
|
||||
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
|
||||
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
|
||||
await assertAgentRunSlotFree(tx, run);
|
||||
});
|
||||
const locked = await claimNativeReviewExecutionLock(tx as unknown as Db, {
|
||||
companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id,
|
||||
|
||||
Reference in new issue
Block a user