mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-20 09:50:48 +01:00
Own the issue admission lock and dispatch after commit in wake-queue
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
21 files changed
+440
-363
No files matched your search
@@ -313,13 +313,16 @@ and shared execution sets so separate service instances keep the same ownership
|
||||
and shutdown barriers. Cleanup single-flight state stays at module scope.
|
||||
Keep recovery policy changes separate from retry scheduling and execution changes.
|
||||
|
||||
Queue admission and wakeup dispatch are in `server/src/services/heartbeat/queue.ts`.
|
||||
It owns wake coalescing and batching, queued-run claims, daily heartbeat caps,
|
||||
concurrency and priority checks, timer admission, and native status wake intents.
|
||||
The wake-queue module admits fresh task runs inside heartbeat's issue admission
|
||||
transaction. It selects eligible saved comment receipts in request order and
|
||||
binds them to the new run. Heartbeat supplies responsible-user resolution and
|
||||
records any specialized continuation delivery before those receipts are bound.
|
||||
Heartbeat coordinates wakeup dispatch, queued-run claims, daily heartbeat caps,
|
||||
concurrency and priority checks, timer admission, and native status wake intents
|
||||
in `server/src/services/heartbeat/queue.ts`. The wake-queue module owns the
|
||||
company and issue lock, the admission transaction, and the handoff after commit
|
||||
for both fresh task runs and wakes arriving behind an active execution. Heartbeat
|
||||
uses the locked transaction for its specialized eligibility gates and continuation
|
||||
writes. Wake-queue selects eligible saved comment receipts in request order and
|
||||
binds them to the new run. Chat authorization and its admission callback run in
|
||||
the same transaction. Authorization contention defers the wake and rolls back
|
||||
without publishing buffered effects.
|
||||
`createHeartbeatQueue` binds these operations without querying or starting work.
|
||||
The service supplies lifecycle effects, execution callbacks, worktree gates, and
|
||||
its process-wide execution and wakeup promise sets. Forwarding callbacks preserve
|
||||
|
||||
@@ -86,8 +86,7 @@ describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing o
|
||||
mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake },
|
||||
recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined },
|
||||
});
|
||||
callbacks.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope);
|
||||
callbacks.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution);
|
||||
callbacks.wakeQueue.admitIssueWake.mockImplementation(admission.admitIssueWake);
|
||||
const queue = createHeartbeatQueue(db, callbacks);
|
||||
const cancel = createStalledQueuedRunCancel(callbacks);
|
||||
const sweep = (beforeCancel?: () => Promise<void>) => assignedWorkSweep(db, {
|
||||
|
||||
@@ -16,7 +16,7 @@ import { settleUnrecoverableExecutions } from "../services/execution-recovery-re
|
||||
import { recordRunPresentationDecision } from "../services/heartbeat/run-completion.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { reconcilePersistedRuntimeServicesOnStartup, readWorkspaceRuntimeIdleWork } from "../services/workspace-runtime.js";
|
||||
import { createAdmissionTransactionScope, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { chatFailedRetryNoticeWork } from "../services/chat-failed-retry-notice-work.js";
|
||||
import { issueTreeControlService } from "../services/issue-tree-control.js";
|
||||
import { chatInboundWakeNoticeWork } from "../services/chat-inbound-wake-notice-work.js";
|
||||
@@ -69831,8 +69831,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
} else if (mode === "requester merge") {
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`select id from issues where id = ${context.issue.id} for update`);
|
||||
await createWakeAdmissionWriter().mergeIntoExistingDeferredWake(
|
||||
createAdmissionTransactionScope(context.fixture.companyId, tx as unknown as Db), {
|
||||
await createWakeAdmissionWriter(tx, context.fixture.companyId).mergeIntoExistingDeferredWake({
|
||||
companyId: context.fixture.companyId, existingDeferredWakeId: originalReceipt.id,
|
||||
mergedPayload: originalReceipt.payload ?? {}, nextCoalescedCount: 1,
|
||||
manualUserWakeActorId: originalReceipt.requestedByActorId!,
|
||||
|
||||
@@ -14,7 +14,7 @@ import { questionResponseDeliveryService } from "../services/question-response-d
|
||||
import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js";
|
||||
import { createDiscardQueuedComment, createReorderQueuedComments } from "../modules/wake-queue/application/queued-comment-use-cases.js";
|
||||
import { createRunDispatch } from "../modules/run-dispatch/index.js";
|
||||
import { createAdmissionTransactionScope, createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
|
||||
@@ -335,8 +335,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
const seeded = await run(f, { contextSnapshot: { issueId: f.issueId, source: "issue_assigned" } });
|
||||
await start(); await flush(); expect(await publications()).toEqual([]); await quiet();
|
||||
await db.transaction(async tx => {
|
||||
await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(
|
||||
createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), {
|
||||
await createWakeAdmissionWriter(tx, f.companyId).coalesceIntoActiveExecutionRun({
|
||||
companyId: f.companyId, activeExecutionRunId: seeded.id, activeExecutionRunStatus: seeded.status, mergedContextSnapshot: chatContextSnapshot(),
|
||||
agentId: f.agentId, source: "on_demand", triggerDetail: null, payload: { issueId: f.issueId },
|
||||
requestedByActorType: "user", requestedByActorId: f.userId, idempotencyKey: null,
|
||||
|
||||
@@ -3,10 +3,11 @@ import { heartbeatRuns } from "@paperclipai/db";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { vi, type Mock, type Mocked } from "vitest";
|
||||
import { createRunDispatch } from "../../modules/run-dispatch/index.js";
|
||||
import type { createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { filterZombieCoalesceTarget } from "../../services/heartbeat.js";
|
||||
import { createHeartbeatRunState } from "../../services/heartbeat/run-state.js";
|
||||
import { createHeartbeatRunPreparation } from "../../services/heartbeat/run-preparation.js";
|
||||
import type { HeartbeatQueueDependencies } from "../../services/heartbeat/queue.js";
|
||||
import { createHeartbeatRunPreparation, mergeCoalescedContextSnapshot } from "../../services/heartbeat/run-preparation.js";
|
||||
import { shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "../../services/heartbeat/queue.js";
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
type MockedCallback = "getAgent" | "setRunStatusFromLive" | "setWakeupStatus" | "appendRunEvent" | "finalizeAgentStatus" | "getAgentInvokability" | "getWorktreeExecutionCutoff" | "applyRunDispatchPostCommitEffects" | "getSchedulingSuppression" | "canStartNewExternalBotWork" | "executeRun" | "applyWakeQueuePostCommitEffects" | "releaseIssueExecutionAndPromote" | "publishRunLifecyclePluginEvent" | "cancelRunInternal" | "cancelActiveForAgentInternal" | "resumeExecutionWaitComments" | "resumeQueuedCommentInterrupt" | "resumeSavedLegacyComments" | "sweepPendingCleanupLeases";
|
||||
@@ -22,6 +23,14 @@ type QueueTestDependencies = HeartbeatQueueDependencies & { [K in MockedCallback
|
||||
export function createHeartbeatQueueTestDependencies(db: Db): QueueTestDependencies {
|
||||
const state = createHeartbeatRunState(db);
|
||||
const preparation = createHeartbeatRunPreparation(db);
|
||||
const admission = createWakeQueue(db, {
|
||||
resolveResponsibleUserId: async () => { throw new Error("Unexpected wake promotion"); },
|
||||
getRoutineEnv: async () => { throw new Error("Unexpected wake promotion"); },
|
||||
resolveSessionBeforeForWakeup: async () => null,
|
||||
wakeAdmissionHelpers: { filterZombieCoalesceTarget, mergeCoalescedContextSnapshot,
|
||||
shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake },
|
||||
recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined },
|
||||
});
|
||||
return {
|
||||
...preparation,
|
||||
...state,
|
||||
@@ -53,8 +62,7 @@ export function createHeartbeatQueueTestDependencies(db: Db): QueueTestDependenc
|
||||
instanceSettings: { getExperimental: vi.fn<HeartbeatQueueDependencies["instanceSettings"]["getExperimental"]>() },
|
||||
runDispatch: createRunDispatch(db),
|
||||
wakeQueue: {
|
||||
createAdmissionTransactionScope: vi.fn<ReturnType<typeof createWakeQueue>["createAdmissionTransactionScope"]>(),
|
||||
admitWakeBehindIssueExecution: vi.fn<ReturnType<typeof createWakeQueue>["admitWakeBehindIssueExecution"]>(),
|
||||
admitIssueWake: vi.fn(admission.admitIssueWake),
|
||||
},
|
||||
activeRunExecutions: new Set<string>(),
|
||||
activeRunExecutionPromises: new Set<Promise<void>>(),
|
||||
|
||||
@@ -12,7 +12,7 @@ import { createChatPublicationResolution } from "../services/chat-channels/publi
|
||||
import { createChatPublicationTaskControl } from "../services/chat-channels/publications/task-control.js";
|
||||
import { chatCredentialMutationLease } from "../services/chat-credential-mutation-lease.js";
|
||||
import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js";
|
||||
import { createAdmissionTransactionScope, createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { applyRunnerGoalPrpEvent } from "../services/runner-goals.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
@@ -205,7 +205,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
const key = `issue-monitor-recovery:${f.issueId}:${issue!.monitorNextCheckAt!.toISOString()}`;
|
||||
await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.transaction(async tx => {
|
||||
await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), {
|
||||
await createWakeAdmissionWriter(tx, f.companyId).coalesceIntoActiveExecutionRun({
|
||||
companyId: f.companyId, agentId: f.agentId, activeExecutionRunId: f.runId, activeExecutionRunStatus: "running",
|
||||
mergedContextSnapshot: { issueId: f.issueId }, source: "automation", triggerDetail: "system",
|
||||
payload: { issueId: f.issueId }, requestedByActorType: "system", requestedByActorId: null,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { and, asc, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { agentWakeupRequests, chatConversations, chatDeliveries, chatMessageLinks, createDb, heartbeatRuns,
|
||||
issueComments, issues, type Db } from "@paperclipai/db";
|
||||
issueComments, issues } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import { seedBoundChatAuthorization } from "../../../__tests__/helpers/bound-chat-authorization.js";
|
||||
import { authorizeChatConversationForBoundRun } from "../../../services/native-runtime/chat-attachment-reuse.js";
|
||||
@@ -11,7 +11,7 @@ import { saveUndeliveredUserComments } from "../../../services/work-intent-outbo
|
||||
import { queuedCommentIdsFromWakePayload } from "../../../services/issue-queued-comment-queue.js";
|
||||
import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "../application/use-cases.js";
|
||||
import type { WakeAdmissionActiveExecutionRun } from "../application/ports.js";
|
||||
import { createAdmissionTransactionScope, createPostgresWakeQueueAdapter, createWakeAdmissionReader,
|
||||
import { createPostgresWakeQueueAdapter, createWakeAdmissionReader,
|
||||
createWakeAdmissionWriter } from "./postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
@@ -31,17 +31,6 @@ describe.skipIf(!support.supported)("chat wake batches stay within one conversat
|
||||
let otherConversationId: string;
|
||||
let deferOnly = true;
|
||||
|
||||
const admit = createAdmitWakeBehindIssueExecution({
|
||||
reader: createWakeAdmissionReader(),
|
||||
writer: createWakeAdmissionWriter(),
|
||||
helpers: {
|
||||
filterZombieCoalesceTarget: (target) => target,
|
||||
mergeCoalescedContextSnapshot,
|
||||
shouldDeferFollowupWakeForSameIssue: () => deferOnly,
|
||||
shouldQueueFollowupForRunningIssueWake: () => false,
|
||||
},
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("wake-chat-conversation-batch-");
|
||||
db = createDb(database.connectionString);
|
||||
@@ -102,7 +91,16 @@ describe.skipIf(!support.supported)("chat wake batches stay within one conversat
|
||||
async function admitWake(contextSnapshot: Record<string, unknown>, activeExecutionRun?: WakeAdmissionActiveExecutionRun) {
|
||||
const commentId = contextSnapshot.wakeCommentId as string;
|
||||
const chat = contextSnapshot.source === "chat:slack";
|
||||
return db.transaction(async (tx) => admit(createAdmissionTransactionScope(f.companyId, tx as unknown as Db), {
|
||||
return db.transaction((tx) => createAdmitWakeBehindIssueExecution({
|
||||
reader: createWakeAdmissionReader(tx, f.companyId),
|
||||
writer: createWakeAdmissionWriter(tx, f.companyId),
|
||||
helpers: {
|
||||
filterZombieCoalesceTarget: (target) => target,
|
||||
mergeCoalescedContextSnapshot,
|
||||
shouldDeferFollowupWakeForSameIssue: () => deferOnly,
|
||||
shouldQueueFollowupForRunningIssueWake: () => false,
|
||||
},
|
||||
})({
|
||||
companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, agentNameKey: "chat runner",
|
||||
issueExecutionAgentNameKey: null,
|
||||
activeExecutionRun: activeExecutionRun ?? { id: f.runId, agentId: f.agentId, status: "running",
|
||||
|
||||
@@ -26,13 +26,11 @@ import {
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import {
|
||||
createAdmissionTransactionScope,
|
||||
createPostgresWakeQueueAdapter,
|
||||
createWakeAdmissionWriter,
|
||||
} from "./postgres.js";
|
||||
import type { WakeQueuePostgresAdapterDeps } from "./postgres.js";
|
||||
import { createReleaseIssueExecution } from "../application/use-cases.js";
|
||||
import type { TransactionScope } from "../application/ports.js";
|
||||
import { withQueuedCommentIdsInRunContext, withQueuedCommentIdsInWakePayload } from "../../../services/issue-queued-comment-queue.js";
|
||||
|
||||
// Proves the atomicity and company-scope properties the security review
|
||||
@@ -1426,8 +1424,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
}).returning();
|
||||
await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, runId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const operation = db.transaction(async tx => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(
|
||||
createAdmissionTransactionScope(companyId, tx as unknown as Db), {
|
||||
const operation = db.transaction(async tx => createWakeAdmissionWriter(tx, companyId).coalesceIntoActiveExecutionRun({
|
||||
companyId, agentId, activeExecutionRunId: runId, activeExecutionRunStatus: "queued",
|
||||
mergedContextSnapshot: withQueuedCommentIdsInRunContext({ issueId }, [oldComment.id, newComment.id]),
|
||||
source: "on_demand", triggerDetail: "manual", payload: { issueId, commentId: newComment.id },
|
||||
@@ -1463,15 +1460,10 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.companyId, companyId))
|
||||
.orderBy(agentWakeupRequests.id);
|
||||
const writer = createWakeAdmissionWriter();
|
||||
|
||||
await expect(
|
||||
db.transaction(async (tx) => {
|
||||
const scope = createAdmissionTransactionScope(
|
||||
companyId,
|
||||
tx as unknown as Db,
|
||||
);
|
||||
await writer.mergeIntoExistingDeferredWake(scope, {
|
||||
await createWakeAdmissionWriter(tx, companyId).mergeIntoExistingDeferredWake({
|
||||
companyId,
|
||||
existingDeferredWakeId: wakeId,
|
||||
mergedPayload: { issueId, changedByMerge: true },
|
||||
@@ -1512,12 +1504,10 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
const writer = createWakeAdmissionWriter();
|
||||
|
||||
await expect(
|
||||
db.transaction(async (tx) => {
|
||||
const scope = createAdmissionTransactionScope(otherCompanyId, tx as unknown as Db);
|
||||
await writer.mergeIntoExistingDeferredWake(scope, {
|
||||
await createWakeAdmissionWriter(tx, otherCompanyId).mergeIntoExistingDeferredWake({
|
||||
companyId: otherCompanyId,
|
||||
existingDeferredWakeId: wakeId,
|
||||
mergedPayload: { issueId, foo: "bar" },
|
||||
@@ -1543,7 +1533,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
});
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(createAdmissionTransactionScope(companyId, tx as unknown as Db), {
|
||||
await createWakeAdmissionWriter(tx, companyId).coalesceIntoActiveExecutionRun({
|
||||
companyId, activeExecutionRunId: runId, activeExecutionRunStatus: "running", mergedContextSnapshot: { commentId: "fresh-input" },
|
||||
agentId, source: "on_demand", triggerDetail: null, payload: null,
|
||||
requestedByActorType: "user", requestedByActorId: null, idempotencyKey: null,
|
||||
@@ -1565,8 +1555,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
paperclipOutputSecretRedactions: [{ fingerprintSha256: "b".repeat(64), material: { scheme: "synthetic" } }],
|
||||
};
|
||||
const runId = await seedRun({ companyId, agentId, status: "running", contextSnapshot: { taskKey: "issue-1", ...history } });
|
||||
await db.transaction(async (tx) => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(
|
||||
createAdmissionTransactionScope(companyId, tx as unknown as Db), {
|
||||
await db.transaction(async (tx) => createWakeAdmissionWriter(tx, companyId).coalesceIntoActiveExecutionRun({
|
||||
companyId, agentId, activeExecutionRunId: runId, activeExecutionRunStatus: "running",
|
||||
mergedContextSnapshot: { taskKey: "issue-1", commentId: "fresh-input" },
|
||||
source: "on_demand", triggerDetail: null, payload: null,
|
||||
@@ -1583,12 +1572,10 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
const otherCompanyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const runId = await seedRun({ companyId, agentId, status: "running", contextSnapshot: { taskKey: "issue-1" } });
|
||||
const writer = createWakeAdmissionWriter();
|
||||
|
||||
await expect(
|
||||
db.transaction(async (tx) => {
|
||||
const scope = createAdmissionTransactionScope(otherCompanyId, tx as unknown as Db);
|
||||
await writer.coalesceIntoActiveExecutionRun(scope, {
|
||||
await createWakeAdmissionWriter(tx, otherCompanyId).coalesceIntoActiveExecutionRun({
|
||||
companyId: otherCompanyId,
|
||||
activeExecutionRunId: runId,
|
||||
activeExecutionRunStatus: "running",
|
||||
@@ -1610,15 +1597,11 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
expect(wakeRows).toHaveLength(0);
|
||||
});
|
||||
|
||||
// Review test (d): a transaction-port call cannot use the root database
|
||||
// executor. The module rejects a missing scope handle and a mismatched
|
||||
// scope handle, and neither case writes a row.
|
||||
it("rejects a missing transaction scope and a mismatched one, without writing through the root executor", async () => {
|
||||
it("rejects input for another company before writing a deferred receipt", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const otherCompanyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const writer = createWakeAdmissionWriter();
|
||||
const newWakeInput = {
|
||||
companyId,
|
||||
agentId,
|
||||
@@ -1630,14 +1613,9 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
idempotencyKey: null,
|
||||
};
|
||||
|
||||
await expect(
|
||||
writer.insertNewDeferredWake(undefined as unknown as TransactionScope, newWakeInput),
|
||||
).rejects.toThrow(/transaction scope/);
|
||||
|
||||
await expect(
|
||||
db.transaction(async (tx) => {
|
||||
const mismatchedScope = createAdmissionTransactionScope(otherCompanyId, tx as unknown as Db);
|
||||
await writer.insertNewDeferredWake(mismatchedScope, newWakeInput);
|
||||
await createWakeAdmissionWriter(tx, otherCompanyId).insertNewDeferredWake(newWakeInput);
|
||||
}),
|
||||
).rejects.toThrow(/different company/);
|
||||
|
||||
|
||||
@@ -42,7 +42,10 @@ import { legacyExecutionNeedsReconciliationWithEvidence } from "../../../service
|
||||
import {
|
||||
authorizeFailedChatRunRetryWake,
|
||||
FailedChatRunRetryAuthorizationError,
|
||||
withDurableChatAdmission,
|
||||
type DurableChatWakeupRequest,
|
||||
} from "../../../services/durable-chat-wakeup.js";
|
||||
import { isExternalChatWaitAuthorizationContention } from "../../../services/native-runtime/chat-attachment-reuse.js";
|
||||
import { isRetiredExternalChatQuestionSource } from "../../../services/question-response-delivery.js";
|
||||
import { HttpError } from "../../../errors.js";
|
||||
import { evaluateAgentInvokabilityFromDb } from "../../../services/agent-invokability.js";
|
||||
@@ -73,7 +76,6 @@ import {
|
||||
parseObject,
|
||||
readNonEmptyString,
|
||||
} from "../domain/values.js";
|
||||
import { requireTransactionScopeTx, TransactionScope } from "../application/ports.js";
|
||||
import type {
|
||||
DeferredWakeCandidate,
|
||||
InvokableAgentSnapshot,
|
||||
@@ -88,6 +90,7 @@ import type {
|
||||
WakeQueueTransaction,
|
||||
} from "../application/ports.js";
|
||||
import type { FreshIssueRunStore } from "../application/fresh-issue-run.js";
|
||||
import type { IssueWakeAdmissionOutcome } from "../application/issue-wake-admission.js";
|
||||
import type { RunSummary } from "../application/types.js";
|
||||
|
||||
const DEFERRED_WAKE_STATUS = "deferred_issue_execution";
|
||||
@@ -97,6 +100,12 @@ const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_r
|
||||
|
||||
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
|
||||
type IssueRow = typeof issues.$inferSelect;
|
||||
export type IssueAdmissionDbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
export type IssueAdmissionTarget = {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
durableChatRequest?: DurableChatWakeupRequest;
|
||||
};
|
||||
|
||||
function normalizeAgentNameKey(value: string | null | undefined): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
@@ -971,23 +980,17 @@ async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: ReleaseRun, iss
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the temporary transaction-scope handle the admission port needs.
|
||||
* `heartbeat.ts` calls this through `createWakeQueue`'s own wrapper; it
|
||||
* never builds a `TransactionScope` itself.
|
||||
*/
|
||||
export function createAdmissionTransactionScope(companyId: string, tx: Db): TransactionScope {
|
||||
return TransactionScope.create(companyId, tx);
|
||||
function assertAdmissionCompany(admittedCompanyId: string, companyId: string) {
|
||||
if (companyId !== admittedCompanyId) {
|
||||
throw new Error("wake-queue: admission is locked for a different company");
|
||||
}
|
||||
}
|
||||
|
||||
function requireAdmissionTx(scope: TransactionScope | null | undefined, companyId: string): Db {
|
||||
return requireTransactionScopeTx(scope, companyId) as Db;
|
||||
}
|
||||
|
||||
export function createWakeAdmissionReader(): WakeAdmissionReader {
|
||||
export function createWakeAdmissionReader(rawTx: IssueAdmissionDbTransaction, admittedCompanyId: string): WakeAdmissionReader {
|
||||
const tx = rawTx as unknown as Db;
|
||||
return {
|
||||
async matchesActiveWakeActor(scope, input) {
|
||||
const tx = requireAdmissionTx(scope, input.companyId);
|
||||
async matchesActiveWakeActor(input) {
|
||||
assertAdmissionCompany(admittedCompanyId, input.companyId);
|
||||
if (!input.wakeupRequestId) return false;
|
||||
const actor = await tx
|
||||
.select({
|
||||
@@ -1010,7 +1013,6 @@ export function createWakeAdmissionReader(): WakeAdmissionReader {
|
||||
);
|
||||
},
|
||||
async isSameExecutionAgent(
|
||||
scope,
|
||||
{
|
||||
companyId,
|
||||
activeExecutionRunAgentId,
|
||||
@@ -1018,7 +1020,7 @@ export function createWakeAdmissionReader(): WakeAdmissionReader {
|
||||
agentNameKey,
|
||||
},
|
||||
) {
|
||||
const tx = requireAdmissionTx(scope, companyId);
|
||||
assertAdmissionCompany(admittedCompanyId, companyId);
|
||||
const executionAgent = await tx
|
||||
.select({ name: agents.name })
|
||||
.from(agents)
|
||||
@@ -1038,10 +1040,9 @@ export function createWakeAdmissionReader(): WakeAdmissionReader {
|
||||
},
|
||||
|
||||
async findExistingDeferredWake(
|
||||
scope,
|
||||
{ companyId, agentId, issueId, chatConversationId, durableActor },
|
||||
) {
|
||||
const tx = requireAdmissionTx(scope, companyId);
|
||||
assertAdmissionCompany(admittedCompanyId, companyId);
|
||||
const row = await tx
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
@@ -1098,10 +1099,11 @@ export function createWakeAdmissionReader(): WakeAdmissionReader {
|
||||
};
|
||||
}
|
||||
|
||||
export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
export function createWakeAdmissionWriter(rawTx: IssueAdmissionDbTransaction, admittedCompanyId: string): WakeAdmissionWriter {
|
||||
const tx = rawTx as unknown as Db;
|
||||
return {
|
||||
async coalesceIntoActiveExecutionRun(scope, input) {
|
||||
const tx = requireAdmissionTx(scope, input.companyId);
|
||||
async coalesceIntoActiveExecutionRun(input) {
|
||||
assertAdmissionCompany(admittedCompanyId, input.companyId);
|
||||
const now = new Date();
|
||||
const [candidate] = await tx.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, input.activeExecutionRunId), eq(heartbeatRuns.companyId, input.companyId),
|
||||
@@ -1173,8 +1175,8 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
return mergedRun as unknown as Record<string, unknown>;
|
||||
},
|
||||
|
||||
async mergeIntoExistingDeferredWake(scope, input) {
|
||||
const tx = requireAdmissionTx(scope, input.companyId);
|
||||
async mergeIntoExistingDeferredWake(input) {
|
||||
assertAdmissionCompany(admittedCompanyId, input.companyId);
|
||||
if (input.manualUserWakeActorId) await notifyChatInboundWakeNotices(tx, { companyId: input.companyId });
|
||||
const [prior] = await tx.select().from(agentWakeupRequests).where(and(
|
||||
eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.id, input.existingDeferredWakeId),
|
||||
@@ -1256,8 +1258,8 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
}
|
||||
},
|
||||
|
||||
async insertNewDeferredWake(scope, input) {
|
||||
const tx = requireAdmissionTx(scope, input.companyId);
|
||||
async insertNewDeferredWake(input) {
|
||||
assertAdmissionCompany(admittedCompanyId, input.companyId);
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
...input.durableReceipt,
|
||||
@@ -1276,6 +1278,32 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
};
|
||||
}
|
||||
|
||||
export function createPostgresIssueWakeAdmission(db: Db) {
|
||||
return async function withIssueAdmission(
|
||||
{ companyId, issueId, durableChatRequest }: IssueAdmissionTarget,
|
||||
evaluate: (tx: IssueAdmissionDbTransaction, stores: {
|
||||
reader: WakeAdmissionReader;
|
||||
writer: WakeAdmissionWriter;
|
||||
fresh: FreshIssueRunStore<HeartbeatRunRow>;
|
||||
}) => Promise<IssueWakeAdmissionOutcome<HeartbeatRunRow>>,
|
||||
): Promise<IssueWakeAdmissionOutcome<HeartbeatRunRow> | null> {
|
||||
try {
|
||||
return await db.transaction(withDurableChatAdmission(durableChatRequest, async (tx) => {
|
||||
// Mentions may target another agent, but every admission must share the issue lock.
|
||||
await tx.execute(sql`select id from issues where id = ${issueId} and company_id = ${companyId} for update`);
|
||||
return evaluate(tx, {
|
||||
reader: createWakeAdmissionReader(tx, companyId),
|
||||
writer: createWakeAdmissionWriter(tx, companyId),
|
||||
fresh: createFreshIssueRunStore(tx, companyId),
|
||||
});
|
||||
}));
|
||||
} catch (error) {
|
||||
if (isExternalChatWaitAuthorizationContention(error)) return null;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter {
|
||||
return {
|
||||
async withIssueExecutionLock(input, fn): Promise<ReleaseTransactionResult & { run: RunSnapshot }> {
|
||||
@@ -1539,10 +1567,10 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
}
|
||||
|
||||
export function createFreshIssueRunStore(
|
||||
scope: TransactionScope,
|
||||
rawTx: IssueAdmissionDbTransaction,
|
||||
companyId: string,
|
||||
): FreshIssueRunStore<HeartbeatRunRow> {
|
||||
const tx = requireAdmissionTx(scope, companyId);
|
||||
const tx = rawTx as unknown as Db;
|
||||
return {
|
||||
async saveReceipt(input) {
|
||||
if (input.kind === "insert") {
|
||||
|
||||
@@ -103,7 +103,7 @@ function isOrdinaryBoardComment(wake: DeferredCommentCandidate): boolean {
|
||||
queuedCommentIdsFromWakePayload(payload).length > 0;
|
||||
}
|
||||
|
||||
/** Admit one new run and its saved inputs using the caller's existing issue lock. */
|
||||
/** Admit one new run and its saved inputs under the issue admission lock. */
|
||||
export function createFreshIssueRunAdmission<Run extends { id: string }>(store: FreshIssueRunStore<Run>) {
|
||||
return async function admitFreshIssueRun(input: FreshIssueRunInput): Promise<Run | undefined> {
|
||||
const { receipt, durableReceipt, durableChatReceiptId, executionWaitRequestId, issue, contextSnapshot } = input;
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { admitIssueWake, type IssueWakeAdmissionEffects, type IssueWakeAdmissionOutcome } from "./issue-wake-admission.js";
|
||||
|
||||
const queuedRun = { id: "run", companyId: "company", agentId: "agent", status: "queued",
|
||||
invocationSource: "assignment", triggerDetail: null, wakeupRequestId: "wake" };
|
||||
type Run = typeof queuedRun;
|
||||
const runningRun = { ...queuedRun, status: "running" };
|
||||
|
||||
function recordEffects(trace: string[]): IssueWakeAdmissionEffects<Run> {
|
||||
return {
|
||||
saved: () => { trace.push("saved"); },
|
||||
publishIssueActivity: () => { trace.push("issue activity"); },
|
||||
executeIssueActions: async () => { trace.push("issue actions"); },
|
||||
cancelledRuns: [{ ...queuedRun, status: "cancelled" }],
|
||||
emitCancelledRun: (run) => {
|
||||
trace.push(run.status);
|
||||
return new Promise(() => {});
|
||||
},
|
||||
readRun: async (runId) => { trace.push(`read ${runId}`); return runningRun; },
|
||||
publishQueuedRun: () => { trace.push("queued event"); },
|
||||
attachSetupRequest: async () => { trace.push("attach setup"); },
|
||||
dispatch: async () => { trace.push("dispatch"); },
|
||||
};
|
||||
}
|
||||
|
||||
describe("issue wake admission handoff", () => {
|
||||
it("waits for commit before publishing or dispatching and leaves telemetry in the background", async () => {
|
||||
const trace: string[] = [];
|
||||
let commit!: (outcome: IssueWakeAdmissionOutcome<Run>) => void;
|
||||
const transaction = new Promise<IssueWakeAdmissionOutcome<Run>>((resolve) => { commit = resolve; });
|
||||
const result = admitIssueWake(() => transaction, recordEffects(trace));
|
||||
expect(trace).toEqual([]);
|
||||
|
||||
commit({ kind: "queued", run: queuedRun });
|
||||
await expect(result).resolves.toBe(queuedRun);
|
||||
expect(trace).toEqual(["saved", "issue activity", "issue actions", "cancelled", "queued event", "attach setup", "dispatch"]);
|
||||
});
|
||||
|
||||
it.each<{ outcome: IssueWakeAdmissionOutcome<Run>; returned: Run | null; handoff: string[] }>([
|
||||
{ outcome: { kind: "coalesced", run: runningRun }, returned: runningRun, handoff: ["attach setup", "dispatch"] },
|
||||
{ outcome: { kind: "replayed", run: queuedRun }, returned: queuedRun, handoff: ["attach setup", "dispatch"] },
|
||||
{ outcome: { kind: "replayed", run: runningRun }, returned: runningRun, handoff: [] },
|
||||
{ outcome: { kind: "durable", receipt: { runId: "run" } }, returned: runningRun, handoff: ["read run"] },
|
||||
{ outcome: { kind: "durable", receipt: { runId: null } }, returned: null, handoff: [] },
|
||||
{ outcome: { kind: "deferred", admitted: true }, returned: null, handoff: [] },
|
||||
{ outcome: { kind: "skipped" }, returned: null, handoff: [] },
|
||||
])("hands off $outcome.kind with the persisted run or receipt", async ({ outcome, returned, handoff }) => {
|
||||
const trace: string[] = [];
|
||||
await expect(admitIssueWake(async () => outcome, recordEffects(trace))).resolves.toBe(returned);
|
||||
const saved = outcome.kind === "deferred" || outcome.kind === "skipped" ? [] : ["saved"];
|
||||
expect(trace).toEqual([...saved, "issue activity", "issue actions", "cancelled", ...handoff]);
|
||||
});
|
||||
|
||||
it("discards all buffered work when commit rejects", async () => {
|
||||
const trace: string[] = [];
|
||||
const error = new Error("commit failed");
|
||||
await expect(admitIssueWake(() => Promise.reject(error), recordEffects(trace))).rejects.toBe(error);
|
||||
expect(trace).toEqual([]);
|
||||
});
|
||||
|
||||
it("surfaces post-commit failures even when they look like authorization contention", async () => {
|
||||
const trace: string[] = [];
|
||||
const effects = recordEffects(trace);
|
||||
const error = Object.assign(new Error("post-commit failure"), { code: "55P03" });
|
||||
effects.executeIssueActions = async () => { throw error; };
|
||||
await expect(admitIssueWake(async () => ({ kind: "queued", run: queuedRun }), effects)).rejects.toBe(error);
|
||||
expect(trace).toEqual(["saved", "issue activity"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { RunSummary } from "./types.js";
|
||||
|
||||
export type IssueWakeAdmissionOutcome<Run> =
|
||||
| { kind: "queued" | "coalesced" | "replayed"; run: Run }
|
||||
| { kind: "durable"; receipt: { runId: string | null } }
|
||||
| { kind: "deferred"; admitted?: boolean }
|
||||
| { kind: "skipped" };
|
||||
|
||||
export type IssueWakeAdmissionEffects<Run> = {
|
||||
saved?(): void;
|
||||
publishIssueActivity(): void;
|
||||
executeIssueActions(): Promise<void>;
|
||||
cancelledRuns: readonly Run[];
|
||||
emitCancelledRun(run: Run): Promise<unknown>;
|
||||
readRun(runId: string): Promise<Run | null>;
|
||||
publishQueuedRun(run: RunSummary): void;
|
||||
attachSetupRequest(): Promise<unknown>;
|
||||
dispatch(): Promise<unknown>;
|
||||
};
|
||||
|
||||
/** Hand off only a committed admission. A contended authorization rolls back. */
|
||||
export async function admitIssueWake<Run extends RunSummary & { status: string }>(
|
||||
commit: () => Promise<IssueWakeAdmissionOutcome<Run> | null>,
|
||||
effects: IssueWakeAdmissionEffects<Run>,
|
||||
): Promise<Run | null> {
|
||||
const outcome = await commit();
|
||||
if (!outcome) return null;
|
||||
|
||||
if (outcome.kind !== "deferred" && outcome.kind !== "skipped") effects.saved?.();
|
||||
effects.publishIssueActivity();
|
||||
await effects.executeIssueActions();
|
||||
|
||||
// Cancellation telemetry does not gate dispatch or the admission result.
|
||||
for (const run of effects.cancelledRuns) void effects.emitCancelledRun(run);
|
||||
|
||||
if (outcome.kind === "durable") {
|
||||
return outcome.receipt.runId ? effects.readRun(outcome.receipt.runId) : null;
|
||||
}
|
||||
if (outcome.kind === "deferred" || outcome.kind === "skipped") return null;
|
||||
|
||||
if (outcome.kind === "queued") effects.publishQueuedRun(outcome.run);
|
||||
if (outcome.kind !== "replayed" || outcome.run.status === "queued") {
|
||||
await effects.attachSetupRequest();
|
||||
await effects.dispatch();
|
||||
}
|
||||
return outcome.run;
|
||||
}
|
||||
@@ -305,49 +305,6 @@ export interface RecoveryEscalationPort {
|
||||
|
||||
export type { PostCommitEffect, ReleaseOutcome };
|
||||
|
||||
/**
|
||||
* Temporary port: `heartbeat.ts` still opens and owns the transaction that
|
||||
* admits a wake behind an active issue execution; the module does not own
|
||||
* that transaction yet. A later change will decompose `enqueueWakeup` so
|
||||
* the module owns the transaction itself. That change removes this handle
|
||||
* and replaces it with a transaction the module opens on its own.
|
||||
*
|
||||
* Only this module builds a scope. `createWakeQueue` exposes a method that
|
||||
* builds one for a caller outside the module. That caller receives an
|
||||
* opaque handle back and never touches this class directly.
|
||||
*/
|
||||
export class TransactionScope {
|
||||
private constructor(
|
||||
private readonly companyId: string,
|
||||
private readonly rawTx: unknown,
|
||||
) {}
|
||||
|
||||
static create(companyId: string, rawTx: unknown): TransactionScope {
|
||||
return new TransactionScope(companyId, rawTx);
|
||||
}
|
||||
|
||||
/** Returns the bound transaction only when `companyId` matches the scope's own company. */
|
||||
requireTx(companyId: string): unknown {
|
||||
if (companyId !== this.companyId) {
|
||||
throw new Error(
|
||||
"wake-queue: the transaction scope belongs to a different company than the requested write",
|
||||
);
|
||||
}
|
||||
return this.rawTx;
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads a scope's bound transaction. Rejects a missing scope with the same clear error as a mismatched one; never falls back to any other executor. */
|
||||
export function requireTransactionScopeTx(
|
||||
scope: TransactionScope | null | undefined,
|
||||
companyId: string,
|
||||
): unknown {
|
||||
if (!scope) {
|
||||
throw new Error("wake-queue: this call carries no transaction scope");
|
||||
}
|
||||
return scope.requireTx(companyId);
|
||||
}
|
||||
|
||||
/** The active execution run a new wake arrives behind. */
|
||||
export type WakeAdmissionActiveExecutionRun = {
|
||||
id: string;
|
||||
@@ -422,11 +379,10 @@ export type AdmitWakeBehindIssueExecutionResult =
|
||||
| { kind: "coalesced"; run: Record<string, unknown> }
|
||||
| { kind: "deferred" };
|
||||
|
||||
/** Read-only lookups the admission use case needs, each scoped to a company. */
|
||||
/** Read-only admission lookups bound to the issue admission transaction. */
|
||||
export interface WakeAdmissionReader {
|
||||
/** A durable incoming request may share an active run only with the exact actor on its persisted wake receipt. */
|
||||
matchesActiveWakeActor(
|
||||
scope: TransactionScope,
|
||||
input: {
|
||||
companyId: string;
|
||||
wakeupRequestId: string | null;
|
||||
@@ -436,7 +392,6 @@ export interface WakeAdmissionReader {
|
||||
): Promise<boolean>;
|
||||
/** True when the active execution run's agent and this wake's own agent share an execution-agent-name key. */
|
||||
isSameExecutionAgent(
|
||||
scope: TransactionScope,
|
||||
input: {
|
||||
companyId: string;
|
||||
activeExecutionRunAgentId: string;
|
||||
@@ -445,7 +400,6 @@ export interface WakeAdmissionReader {
|
||||
},
|
||||
): Promise<boolean>;
|
||||
findExistingDeferredWake(
|
||||
scope: TransactionScope,
|
||||
input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
@@ -461,7 +415,6 @@ export interface WakeAdmissionReader {
|
||||
export interface WakeAdmissionWriter {
|
||||
/** Merges the wake's context into the active execution run and records the wake as coalesced. Returns the updated run row. */
|
||||
coalesceIntoActiveExecutionRun(
|
||||
scope: TransactionScope,
|
||||
input: {
|
||||
companyId: string;
|
||||
activeExecutionRunId: string;
|
||||
@@ -479,7 +432,6 @@ export interface WakeAdmissionWriter {
|
||||
): Promise<Record<string, unknown> | null>;
|
||||
/** Merges the wake's context into an already-queued deferred wake, guarded by its current status. */
|
||||
mergeIntoExistingDeferredWake(
|
||||
scope: TransactionScope,
|
||||
input: {
|
||||
companyId: string;
|
||||
existingDeferredWakeId: string;
|
||||
@@ -493,7 +445,6 @@ export interface WakeAdmissionWriter {
|
||||
): Promise<void>;
|
||||
/** Queues a new deferred wake behind the active execution run. */
|
||||
insertNewDeferredWake(
|
||||
scope: TransactionScope,
|
||||
input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
|
||||
@@ -11,7 +11,6 @@ import type {
|
||||
RecoveryEscalationPort,
|
||||
RunSnapshot,
|
||||
RunSummary,
|
||||
TransactionScope,
|
||||
WakeAdmissionActiveExecutionRun,
|
||||
WakeAdmissionHeartbeatHelpers,
|
||||
WakeAdmissionReader,
|
||||
@@ -914,9 +913,6 @@ const ACTIVE_EXECUTION_RUN: WakeAdmissionActiveExecutionRun = {
|
||||
contextSnapshot: { taskKey: "issue-1" },
|
||||
};
|
||||
|
||||
// The scope is opaque to the use case; the fakes below never inspect it.
|
||||
const SCOPE = {} as TransactionScope;
|
||||
|
||||
function admissionInput(
|
||||
overrides: Partial<AdmitWakeBehindIssueExecutionInput> = {},
|
||||
): AdmitWakeBehindIssueExecutionInput {
|
||||
@@ -1011,7 +1007,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
};
|
||||
|
||||
const result = await admit(
|
||||
SCOPE,
|
||||
admissionInput({
|
||||
activeExecutionRun: { ...ACTIVE_EXECUTION_RUN, status },
|
||||
allowRunCoalescing: false,
|
||||
@@ -1025,7 +1020,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
expect(reader.findExistingDeferredWake).not.toHaveBeenCalled();
|
||||
expect(writer.mergeIntoExistingDeferredWake).not.toHaveBeenCalled();
|
||||
expect(writer.insertNewDeferredWake).toHaveBeenCalledExactlyOnceWith(
|
||||
SCOPE,
|
||||
expect.objectContaining({
|
||||
durableReceipt,
|
||||
payload: {
|
||||
@@ -1047,7 +1041,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
} : null),
|
||||
});
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers() });
|
||||
await admit(SCOPE, admissionInput({
|
||||
await admit(admissionInput({
|
||||
contextSnapshot: location === "incoming" ? { interactionId: "approval" } : {},
|
||||
activeExecutionRun: { ...ACTIVE_EXECUTION_RUN, status: "queued",
|
||||
contextSnapshot: location === "queued" ? { interactionId: "approval" } : {} },
|
||||
@@ -1066,7 +1060,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
const writer = createFakeAdmissionWriter();
|
||||
const reader = createFakeAdmissionReader();
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers() });
|
||||
await admit(SCOPE, admissionInput({
|
||||
await admit(admissionInput({
|
||||
contextSnapshot: incoming ? { chatConversationId: incoming } : {},
|
||||
activeExecutionRun: { ...ACTIVE_EXECUTION_RUN, status: "queued",
|
||||
contextSnapshot: active ? { chatConversationId: active } : {} },
|
||||
@@ -1074,7 +1068,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
expect(writer.coalesceIntoActiveExecutionRun).toHaveBeenCalledTimes(coalesces ? 1 : 0);
|
||||
if (!coalesces) {
|
||||
// The separated reply still merges with its own conversation's deferred wake.
|
||||
expect(reader.findExistingDeferredWake).toHaveBeenCalledExactlyOnceWith(SCOPE,
|
||||
expect(reader.findExistingDeferredWake).toHaveBeenCalledExactlyOnceWith(
|
||||
expect.objectContaining({ chatConversationId: incoming }));
|
||||
expect(writer.insertNewDeferredWake).toHaveBeenCalledOnce();
|
||||
}
|
||||
@@ -1097,7 +1091,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
});
|
||||
|
||||
const result = await admit(
|
||||
SCOPE,
|
||||
admissionInput({
|
||||
activeExecutionRun: {
|
||||
...ACTIVE_EXECUTION_RUN,
|
||||
@@ -1111,7 +1104,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
|
||||
expect(result).toEqual({ kind: "deferred" });
|
||||
expect(reader.matchesActiveWakeActor).toHaveBeenCalledExactlyOnceWith(
|
||||
SCOPE,
|
||||
{
|
||||
companyId: "company-1",
|
||||
wakeupRequestId: "active-wake",
|
||||
@@ -1121,7 +1113,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
);
|
||||
expect(writer.coalesceIntoActiveExecutionRun).not.toHaveBeenCalled();
|
||||
expect(reader.findExistingDeferredWake).toHaveBeenCalledExactlyOnceWith(
|
||||
SCOPE,
|
||||
{
|
||||
companyId: "company-1",
|
||||
agentId: "wake-agent",
|
||||
@@ -1131,7 +1122,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
},
|
||||
);
|
||||
expect(writer.insertNewDeferredWake).toHaveBeenCalledExactlyOnceWith(
|
||||
SCOPE,
|
||||
expect.objectContaining({ durableReceipt }),
|
||||
);
|
||||
});
|
||||
@@ -1151,7 +1141,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
});
|
||||
|
||||
const result = await admit(
|
||||
SCOPE,
|
||||
admissionInput({
|
||||
activeExecutionRun: {
|
||||
...ACTIVE_EXECUTION_RUN,
|
||||
@@ -1163,7 +1152,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
|
||||
expect(result).toEqual({ kind: "coalesced", run: { id: "merged-run-1" } });
|
||||
expect(reader.matchesActiveWakeActor).toHaveBeenCalledExactlyOnceWith(
|
||||
SCOPE,
|
||||
{
|
||||
companyId: "company-1",
|
||||
wakeupRequestId: "active-wake",
|
||||
@@ -1174,7 +1162,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
expect(
|
||||
writer.coalesceIntoActiveExecutionRun,
|
||||
).toHaveBeenCalledExactlyOnceWith(
|
||||
SCOPE,
|
||||
expect.objectContaining({ durableReceipt }),
|
||||
);
|
||||
expect(reader.findExistingDeferredWake).not.toHaveBeenCalled();
|
||||
@@ -1204,7 +1191,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
});
|
||||
|
||||
const result = await admit(
|
||||
SCOPE,
|
||||
admissionInput({
|
||||
durableReceipt,
|
||||
reason: "question_response",
|
||||
@@ -1216,7 +1202,6 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
expect(
|
||||
writer.mergeIntoExistingDeferredWake,
|
||||
).toHaveBeenCalledExactlyOnceWith(
|
||||
SCOPE,
|
||||
expect.objectContaining({
|
||||
existingDeferredWakeId: "existing-deferred-wake",
|
||||
nextCoalescedCount: 3,
|
||||
@@ -1256,10 +1241,10 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
})) });
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer,
|
||||
helpers: createFakeAdmissionHelpers({ shouldDeferFollowupWakeForSameIssue: vi.fn(() => true) }) });
|
||||
await expect(admit(SCOPE, admissionInput({ wakeCommentId: "second-comment", requestedByActorId: "second-user",
|
||||
await expect(admit(admissionInput({ wakeCommentId: "second-comment", requestedByActorId: "second-user",
|
||||
payload: { commentId: "second-comment", coalescedIntoWakeupRequestId: "forged-target" },
|
||||
}))).resolves.toEqual({ kind: "deferred" });
|
||||
const input = vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][1];
|
||||
const input = vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][0];
|
||||
expect(input.coalescedReceipt).toEqual({ agentId: "wake-agent", source: "on_demand", triggerDetail: null,
|
||||
reason: null, payload: { issueId: "issue-1", commentId: "second-comment", coalescedIntoWakeupRequestId: "existing-deferred-wake" },
|
||||
requestedByActorType: "user", requestedByActorId: "second-user", idempotencyKey: null, runId: null });
|
||||
@@ -1275,8 +1260,8 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
reader: createFakeAdmissionReader({ findExistingDeferredWake: vi.fn(async () => ({
|
||||
id: "saved-wake", runId: null, payload: { issueId: "issue-1" }, deferredContext: {}, coalescedCount: 0,
|
||||
})) }), helpers: createFakeAdmissionHelpers({ shouldDeferFollowupWakeForSameIssue: vi.fn(() => true) }) });
|
||||
await admit(SCOPE, admissionInput({ durableReceipt, payload: { text: "saved input" } }));
|
||||
expect(vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][1].coalescedReceipt).toEqual(expect.objectContaining({
|
||||
await admit(admissionInput({ durableReceipt, payload: { text: "saved input" } }));
|
||||
expect(vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][0].coalescedReceipt).toEqual(expect.objectContaining({
|
||||
...durableReceipt, payload: { text: "saved input", issueId: "issue-1", coalescedIntoWakeupRequestId: "saved-wake" },
|
||||
}));
|
||||
});
|
||||
@@ -1287,8 +1272,8 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
reader: createFakeAdmissionReader({ findExistingDeferredWake: vi.fn(async () => ({
|
||||
id: "existing-deferred-wake", runId: null, payload: { issueId: "issue-1" }, deferredContext: {}, coalescedCount: 1,
|
||||
})) }), helpers: createFakeAdmissionHelpers({ shouldDeferFollowupWakeForSameIssue: vi.fn(() => true) }) });
|
||||
await admit(SCOPE, admissionInput({ wakeCommentId: "second-comment", requestedByActorType: actor }));
|
||||
expect(vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][1]).not.toHaveProperty("coalescedReceipt");
|
||||
await admit(admissionInput({ wakeCommentId: "second-comment", requestedByActorType: actor }));
|
||||
expect(vi.mocked(writer.mergeIntoExistingDeferredWake).mock.calls[0][0]).not.toHaveProperty("coalescedReceipt");
|
||||
});
|
||||
|
||||
it("gives manual input its own run boundary even when the active receipt has the same requester", async () => {
|
||||
@@ -1296,7 +1281,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
const admit = createAdmitWakeBehindIssueExecution({
|
||||
reader: createFakeAdmissionReader(), writer, helpers: createFakeAdmissionHelpers(),
|
||||
});
|
||||
expect(await admit(SCOPE, admissionInput({ payload: { issueId: "issue-1", manualUserWake: true } })))
|
||||
expect(await admit(admissionInput({ payload: { issueId: "issue-1", manualUserWake: true } })))
|
||||
.toEqual({ kind: "deferred" });
|
||||
expect(writer.coalesceIntoActiveExecutionRun).not.toHaveBeenCalled();
|
||||
expect(writer.insertNewDeferredWake).toHaveBeenCalledTimes(1);
|
||||
@@ -1314,7 +1299,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
helpers,
|
||||
});
|
||||
|
||||
expect(await admit(SCOPE, admissionInput())).toEqual({
|
||||
expect(await admit(admissionInput())).toEqual({
|
||||
kind: "coalesced",
|
||||
run: { id: "merged-run-1" },
|
||||
});
|
||||
@@ -1328,7 +1313,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
const helpers = createFakeAdmissionHelpers();
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers });
|
||||
|
||||
const result = await admit(SCOPE, admissionInput());
|
||||
const result = await admit(admissionInput());
|
||||
|
||||
expect(result).toEqual({ kind: "coalesced", run: { id: "merged-run-1" } });
|
||||
expect(writer.coalesceIntoActiveExecutionRun).toHaveBeenCalledTimes(1);
|
||||
@@ -1342,7 +1327,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
const helpers = createFakeAdmissionHelpers();
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers });
|
||||
|
||||
const result = await admit(SCOPE, admissionInput());
|
||||
const result = await admit(admissionInput());
|
||||
|
||||
expect(result.kind).toBe("coalesced");
|
||||
expect(reader.findExistingDeferredWake).not.toHaveBeenCalled();
|
||||
@@ -1353,10 +1338,10 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
const reader = createFakeAdmissionReader();
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: createFakeAdmissionHelpers() });
|
||||
const durableReceipt = { id: "incoming-source", requestedAt: new Date("2026-11-04T00:00:00Z") };
|
||||
expect(await admit(SCOPE, admissionInput({ durableReceipt, wakeCommentId: "new-comment",
|
||||
expect(await admit(admissionInput({ durableReceipt, wakeCommentId: "new-comment",
|
||||
contextSnapshot: { issueId: "issue-1", wakeCommentId: "new-comment" }, payload: { issueId: "issue-1", commentId: "new-comment" },
|
||||
}))).toEqual({ kind: "deferred" });
|
||||
expect(writer.insertNewDeferredWake).toHaveBeenCalledWith(SCOPE, expect.objectContaining({
|
||||
expect(writer.insertNewDeferredWake).toHaveBeenCalledWith(expect.objectContaining({
|
||||
companyId: "company-1", agentId: "wake-agent", durableReceipt,
|
||||
requestedByActorType: "user", requestedByActorId: "user-1",
|
||||
payload: expect.objectContaining({ issueId: "issue-1", commentId: "new-comment" }),
|
||||
@@ -1366,7 +1351,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
|
||||
it("merges into the existing deferred wake when the policy returns a merge target", async () => {
|
||||
const mergeIntoExistingDeferredWake = vi.fn(
|
||||
async (_scope: TransactionScope, _input: Parameters<WakeAdmissionWriter["mergeIntoExistingDeferredWake"]>[1]) => {},
|
||||
async (_input: Parameters<WakeAdmissionWriter["mergeIntoExistingDeferredWake"]>[0]) => {},
|
||||
);
|
||||
const writer = createFakeAdmissionWriter({ mergeIntoExistingDeferredWake });
|
||||
const reader = createFakeAdmissionReader({
|
||||
@@ -1381,11 +1366,11 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
const helpers = createFakeAdmissionHelpers();
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers });
|
||||
|
||||
const result = await admit(SCOPE, admissionInput());
|
||||
const result = await admit(admissionInput());
|
||||
|
||||
expect(result).toEqual({ kind: "deferred" });
|
||||
expect(mergeIntoExistingDeferredWake).toHaveBeenCalledTimes(1);
|
||||
const call = mergeIntoExistingDeferredWake.mock.calls[0]![1];
|
||||
const call = mergeIntoExistingDeferredWake.mock.calls[0]![0];
|
||||
expect(call.existingDeferredWakeId).toBe("deferred-1");
|
||||
expect(call.nextCoalescedCount).toBe(3);
|
||||
expect(call.mergedPayload.foo).toBe("bar");
|
||||
@@ -1399,7 +1384,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
const helpers = createFakeAdmissionHelpers();
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers });
|
||||
|
||||
const result = await admit(SCOPE, admissionInput());
|
||||
const result = await admit(admissionInput());
|
||||
|
||||
expect(result).toEqual({ kind: "deferred" });
|
||||
expect(writer.insertNewDeferredWake).toHaveBeenCalledTimes(1);
|
||||
@@ -1413,7 +1398,7 @@ describe("admitWakeBehindIssueExecution", () => {
|
||||
const helpers = createFakeAdmissionHelpers();
|
||||
const admit = createAdmitWakeBehindIssueExecution({ reader, writer, helpers });
|
||||
|
||||
const result = await admit(SCOPE, admissionInput({ liveRunExecutions: { has: () => false } }));
|
||||
const result = await admit(admissionInput({ liveRunExecutions: { has: () => false } }));
|
||||
|
||||
expect(result).toEqual({ kind: "proceed" });
|
||||
expect(reader.findExistingDeferredWake).not.toHaveBeenCalled();
|
||||
|
||||
@@ -26,7 +26,6 @@ import type {
|
||||
RecoveryEscalationPort,
|
||||
ReleaseTransactionResult,
|
||||
RunSnapshot,
|
||||
TransactionScope,
|
||||
WakeAdmissionActiveExecutionRun,
|
||||
WakeAdmissionHeartbeatHelpers,
|
||||
WakeAdmissionReader,
|
||||
@@ -868,7 +867,6 @@ export function createAdmitWakeBehindIssueExecution(deps: {
|
||||
helpers: WakeAdmissionHeartbeatHelpers;
|
||||
}) {
|
||||
return async function admitWakeBehindIssueExecution(
|
||||
scope: TransactionScope,
|
||||
input: AdmitWakeBehindIssueExecutionInput,
|
||||
): Promise<AdmitWakeBehindIssueExecutionResult> {
|
||||
const manualUserWakeActorId = input.payload?.manualUserWake === true
|
||||
@@ -877,7 +875,7 @@ export function createAdmitWakeBehindIssueExecution(deps: {
|
||||
(input.requestedByActorType !== "user" || !manualUserWakeActorId)) {
|
||||
throw new Error("wake-queue: manual wake requires an authenticated user");
|
||||
}
|
||||
const isSameExecutionAgent = await deps.reader.isSameExecutionAgent(scope, {
|
||||
const isSameExecutionAgent = await deps.reader.isSameExecutionAgent({
|
||||
companyId: input.companyId,
|
||||
activeExecutionRunAgentId: input.activeExecutionRun.agentId,
|
||||
issueExecutionAgentNameKey: input.issueExecutionAgentNameKey,
|
||||
@@ -909,7 +907,7 @@ export function createAdmitWakeBehindIssueExecution(deps: {
|
||||
|
||||
const sameDurableActor =
|
||||
!input.durableReceipt ||
|
||||
(await deps.reader.matchesActiveWakeActor(scope, {
|
||||
(await deps.reader.matchesActiveWakeActor({
|
||||
companyId: input.companyId,
|
||||
wakeupRequestId: availableActiveExecutionRun?.wakeupRequestId ?? null,
|
||||
requestedByActorType: input.requestedByActorType,
|
||||
@@ -940,7 +938,7 @@ export function createAdmitWakeBehindIssueExecution(deps: {
|
||||
target.status === "queued" || target.status === "scheduled_retry",
|
||||
},
|
||||
);
|
||||
const run = await deps.writer.coalesceIntoActiveExecutionRun(scope, {
|
||||
const run = await deps.writer.coalesceIntoActiveExecutionRun({
|
||||
companyId: input.companyId,
|
||||
activeExecutionRunId: target.id,
|
||||
activeExecutionRunStatus: target.status,
|
||||
@@ -962,7 +960,7 @@ export function createAdmitWakeBehindIssueExecution(deps: {
|
||||
const existingDeferred =
|
||||
(interactionReceipt || input.allowRunCoalescing === false)
|
||||
? null
|
||||
: await deps.reader.findExistingDeferredWake(scope, {
|
||||
: await deps.reader.findExistingDeferredWake({
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
issueId: input.issueId,
|
||||
@@ -995,7 +993,7 @@ export function createAdmitWakeBehindIssueExecution(deps: {
|
||||
issueId: input.issueId,
|
||||
[DEFERRED_WAKE_CONTEXT_KEY]: mergedDeferredContext,
|
||||
};
|
||||
await deps.writer.mergeIntoExistingDeferredWake(scope, {
|
||||
await deps.writer.mergeIntoExistingDeferredWake({
|
||||
companyId: input.companyId,
|
||||
existingDeferredWakeId: existingDeferred.id,
|
||||
mergedPayload,
|
||||
@@ -1031,7 +1029,7 @@ export function createAdmitWakeBehindIssueExecution(deps: {
|
||||
issueId: input.issueId,
|
||||
[DEFERRED_WAKE_CONTEXT_KEY]: input.contextSnapshot,
|
||||
};
|
||||
await deps.writer.insertNewDeferredWake(scope, {
|
||||
await deps.writer.insertNewDeferredWake({
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
source: input.source,
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { Db, heartbeatRuns } from "@paperclipai/db";
|
||||
import {
|
||||
createAdmissionTransactionScope as buildAdmissionTransactionScope,
|
||||
createFreshIssueRunStore,
|
||||
createPostgresIssueWakeAdmission,
|
||||
createPostgresWakeQueueAdapter,
|
||||
createWakeAdmissionReader,
|
||||
createWakeAdmissionWriter,
|
||||
type IssueAdmissionDbTransaction,
|
||||
type IssueAdmissionTarget,
|
||||
} from "./adapters/postgres.js";
|
||||
import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js";
|
||||
import type { QueuedCommentQueuePostgresAdapterDeps } from "./adapters/queued-comment-postgres.js";
|
||||
import { createFreshIssueRunAdmission } from "./application/fresh-issue-run.js";
|
||||
import { admitIssueWake, type IssueWakeAdmissionEffects, type IssueWakeAdmissionOutcome } from "./application/issue-wake-admission.js";
|
||||
import { createAdmitWakeBehindIssueExecution, createReleaseIssueExecution } from "./application/use-cases.js";
|
||||
import {
|
||||
createDiscardQueuedComment,
|
||||
@@ -19,7 +19,6 @@ import type {
|
||||
IssueSnapshot,
|
||||
RecoveryEscalationPort,
|
||||
RunSnapshot,
|
||||
TransactionScope,
|
||||
WakeAdmissionHeartbeatHelpers,
|
||||
WakeQueueHost,
|
||||
} from "./application/ports.js";
|
||||
@@ -37,9 +36,8 @@ export type {
|
||||
RunSnapshot,
|
||||
RecoveryEscalationPort,
|
||||
ReleaseRecoveryBlockedNoticeKind,
|
||||
TransactionScope,
|
||||
} from "./application/ports.js";
|
||||
export type { AdmitWakeBehindIssueExecutionInput, AdmitWakeBehindIssueExecutionResult, ReleaseIssueExecutionInput } from "./application/use-cases.js";
|
||||
export type { ReleaseIssueExecutionInput } from "./application/use-cases.js";
|
||||
export {
|
||||
QueuedCommentMutationError,
|
||||
QueuedCommentMutationForbiddenError,
|
||||
@@ -78,45 +76,53 @@ export type WakeQueueDeps = {
|
||||
recovery: RecoveryEscalationPort;
|
||||
};
|
||||
|
||||
/**
|
||||
* Composes the wake-queue module: the Postgres adapter (which owns the
|
||||
* release transaction) and the release use case. `heartbeat.ts` holds the
|
||||
* only caller: it builds one instance per process next to
|
||||
* `createRunDispatch(db)` and delegates `releaseIssueExecutionAndPromote`'s
|
||||
* body to `releaseIssueExecution`.
|
||||
*
|
||||
* The admission half is temporary: `heartbeat.ts` still opens and owns the
|
||||
* transaction that admits a wake behind an active issue execution, so it
|
||||
* builds a `TransactionScope` through `createAdmissionTransactionScope`
|
||||
* before it calls `admitWakeBehindIssueExecution`.
|
||||
*/
|
||||
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
|
||||
type LockedIssueAdmission = {
|
||||
admitWakeBehindIssueExecution: ReturnType<typeof createAdmitWakeBehindIssueExecution>;
|
||||
admitFreshIssueRun: ReturnType<typeof createFreshIssueRunAdmission<HeartbeatRun>>;
|
||||
};
|
||||
|
||||
/** Composes the issue admission and release workflows with their Postgres locks. */
|
||||
export function createWakeQueue(db: Db, deps: WakeQueueDeps) {
|
||||
const issueLock = createPostgresWakeQueueAdapter(db, {
|
||||
resolveResponsibleUserId: deps.resolveResponsibleUserId,
|
||||
getRoutineEnv: deps.getRoutineEnv,
|
||||
resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup,
|
||||
});
|
||||
const withIssueAdmission = createPostgresIssueWakeAdmission(db);
|
||||
|
||||
return {
|
||||
releaseIssueExecution: createReleaseIssueExecution({ issueLock, recovery: deps.recovery }),
|
||||
admitWakeBehindIssueExecution: createAdmitWakeBehindIssueExecution({
|
||||
reader: createWakeAdmissionReader(),
|
||||
writer: createWakeAdmissionWriter(),
|
||||
helpers: deps.wakeAdmissionHelpers,
|
||||
}),
|
||||
createAdmissionTransactionScope(companyId: string, tx: Db): TransactionScope {
|
||||
return buildAdmissionTransactionScope(companyId, tx);
|
||||
admitIssueWake(
|
||||
{ companyId, issueId, durableChatRequest }: IssueAdmissionTarget,
|
||||
// Specialized heartbeat gates and continuation writes still use this locked transaction.
|
||||
evaluateHeartbeatPolicy: (tx: IssueAdmissionDbTransaction, admission: LockedIssueAdmission) => Promise<IssueWakeAdmissionOutcome<HeartbeatRun>>,
|
||||
effects: IssueWakeAdmissionEffects<HeartbeatRun>,
|
||||
) {
|
||||
return admitIssueWake(() => withIssueAdmission({ companyId, issueId, durableChatRequest }, (tx, { reader, writer, fresh }) => {
|
||||
const admitBusy = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: deps.wakeAdmissionHelpers });
|
||||
const admitFresh = createFreshIssueRunAdmission(fresh);
|
||||
return evaluateHeartbeatPolicy(tx, {
|
||||
async admitWakeBehindIssueExecution(input) {
|
||||
if (input.companyId !== companyId || input.issueId !== issueId || input.contextSnapshot.issueId !== issueId) {
|
||||
throw new Error("wake-queue: admission does not match the locked company and issue");
|
||||
}
|
||||
return admitBusy(input);
|
||||
},
|
||||
async admitFreshIssueRun(input) {
|
||||
if (input.issue.id !== issueId || input.contextSnapshot.issueId !== issueId) {
|
||||
throw new Error("wake-queue: admission does not match the locked issue");
|
||||
}
|
||||
return admitFresh(input);
|
||||
},
|
||||
});
|
||||
}), effects);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type WakeQueue = ReturnType<typeof createWakeQueue>;
|
||||
|
||||
/** Uses the transaction that already owns the issue's admission lock. */
|
||||
export function bindFreshIssueRunAdmission(companyId: string, tx: Db) {
|
||||
return createFreshIssueRunAdmission(createFreshIssueRunStore(buildAdmissionTransactionScope(companyId, tx), companyId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Composes the three queued-comment queue mutations (edit, reorder,
|
||||
* discard): the Postgres adapter, which owns the one transaction each
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { activityLog, agents, agentWakeupRequests, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issueThreadInteractions, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, type Db } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentWakeupRequests, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issueThreadInteractions, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
import { filterRunMessageInteractionDelivery, hasAutomaticInteractionDeliveryHold, preparedRunInteractionDeliveryDenied, recheckQueuedCommentInteractionDelivery,
|
||||
recheckQueuedRunInteractions, withInteractionDeliveryOrigin } from "./agent-interaction-delivery.js";
|
||||
@@ -9,7 +9,7 @@ import { createHeartbeatQueue } from "./heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "../__tests__/helpers/heartbeat-queue.js";
|
||||
import { withPluginInteractionOrigin } from "./plugin-interaction-context.js";
|
||||
import { recoveryService } from "./recovery/service.js";
|
||||
import { createAdmissionTransactionScope, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("pending agent interaction delivery", () => {
|
||||
@@ -113,8 +113,7 @@ describe.skipIf(!support.supported)("pending agent interaction delivery", () =>
|
||||
const context = { issueId: f.issueId, wakeReason: reason, ...(assignmentFirst ? { wakeCommentId: messageId, wakeCommentIds: [messageId] } : {}) };
|
||||
const actorType = assignmentFirst ? "user" : "agent", actorId = assignmentFirst ? "human" : f.sender;
|
||||
const payload = withInteractionDeliveryOrigin({ issueId: f.issueId, _paperclipWakeContext: context }, actorType, actorId);
|
||||
await db.transaction(tx => createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(
|
||||
createAdmissionTransactionScope(f.companyId, tx as unknown as Db), {
|
||||
await db.transaction(tx => createWakeAdmissionWriter(tx, f.companyId).coalesceIntoActiveExecutionRun({
|
||||
companyId: f.companyId, agentId: f.recipient, activeExecutionRunId: f.run.id, activeExecutionRunStatus: "queued",
|
||||
mergedContextSnapshot: { ...context, wakeCommentId: messageId, wakeCommentIds: [messageId] },
|
||||
source: "assignment", triggerDetail: "system", payload, requestedByActorType: actorType,
|
||||
|
||||
@@ -12,11 +12,11 @@ import { createReleaseIssueExecution } from "../../modules/wake-queue/applicatio
|
||||
import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js";
|
||||
import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js";
|
||||
import { createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { TransactionScope } from "../../modules/wake-queue/application/ports.js";
|
||||
import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js";
|
||||
import { withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext } from "../issue-queued-comment-queue.js";
|
||||
import { settleConversationTurn } from "../agent-conversations.js";
|
||||
import { chatCompletionDeliveryService, recordChatCompletion, recordChatHandoff } from "../chat-completion-delivery.js";
|
||||
import { createDurableChatWakeupRequest } from "../durable-chat-wakeup.js";
|
||||
import { instanceSettingsService } from "../instance-settings.js";
|
||||
import { createHeartbeatRunControl } from "./run-control.js";
|
||||
import { callbacks as runControlCallbacks } from "./__tests__/run-control-fixtures.js";
|
||||
@@ -255,6 +255,60 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
return { ...f, agent, service };
|
||||
}
|
||||
|
||||
it.each([
|
||||
["fresh", "authorization contention"], ["busy", "authorization contention"],
|
||||
["fresh", "wrong locked issue"], ["busy", "wrong locked issue"],
|
||||
] as const)("rolls back a %s chat admission on %s without publishing", async (kind, failure) => {
|
||||
const { user, company, agent, issue, deps, queue } = await waitingConversationFixture();
|
||||
await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, conversationState: null, status: "todo" })
|
||||
.where(eq(issues.id, issue.id));
|
||||
if (kind === "busy") {
|
||||
const [active] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,
|
||||
status: "running", invocationSource: "assignment", contextSnapshot: { issueId: issue.id } }).returning();
|
||||
await db.update(issues).set({ executionRunId: active.id, executionAgentNameKey: agent.name, executionLockedAt: new Date() })
|
||||
.where(eq(issues.id, issue.id));
|
||||
deps.liveRunExecutions.has.mockImplementation((id) => id === active.id);
|
||||
}
|
||||
const [comment] = await db.insert(issueComments).values({ companyId: company.id, issueId: issue.id,
|
||||
authorType: "user", authorUserId: user.id, body: "Please continue" }).returning();
|
||||
const id = randomUUID();
|
||||
const authorize = vi.fn(async () => {});
|
||||
const onAdmitted = vi.fn(async (tx: Db, { runId }: { runId: string | null }) => {
|
||||
const [receipt] = await tx.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, id));
|
||||
expect(receipt).toMatchObject({ status: kind === "fresh" ? "queued" : "deferred_issue_execution", runId });
|
||||
expect(Boolean(runId)).toBe(kind === "fresh");
|
||||
throw Object.assign(new Error("authorization lock was held"), { code: "55P03" });
|
||||
});
|
||||
const request = createDurableChatWakeupRequest({ id, companyId: company.id, issueId: issue.id, agentId: agent.id,
|
||||
commentId: comment.id, requestedByActorType: "user", requestedByActorId: user.id, requestedAt: new Date(), authorize, onAdmitted });
|
||||
const [otherIssue] = failure === "wrong locked issue" ? await db.insert(issues).values({
|
||||
companyId: company.id, assigneeAgentId: agent.id, title: "Another task",
|
||||
}).returning() : [];
|
||||
const publishIssueActivity = vi.fn();
|
||||
const admit = deps.wakeQueue.admitIssueWake.getMockImplementation()!;
|
||||
deps.wakeQueue.admitIssueWake.mockImplementation((target, evaluate, effects) => admit(target,
|
||||
(tx, admission) => evaluate(tx, otherIssue ? {
|
||||
admitFreshIssueRun: (input) => admission.admitFreshIssueRun({ ...input,
|
||||
issue: { ...input.issue, id: otherIssue.id }, contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }),
|
||||
admitWakeBehindIssueExecution: (input) => admission.admitWakeBehindIssueExecution({ ...input,
|
||||
issueId: otherIssue.id, contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }),
|
||||
} : admission), { ...effects, publishIssueActivity }));
|
||||
|
||||
const wake = queue.enqueueWakeup(agent.id, {
|
||||
source: "assignment", triggerDetail: "system", reason: "External chat message received",
|
||||
requestedByActorType: "user", requestedByActorId: user.id, allowRunCoalescing: false, durableChatRequest: request,
|
||||
payload: { issueId: issue.id, commentId: comment.id, mutation: "chat_message_received" },
|
||||
});
|
||||
if (otherIssue) await expect(wake).rejects.toThrow(/does not match the locked/);
|
||||
else await expect(wake).resolves.toBeNull();
|
||||
expect(authorize).toHaveBeenCalledTimes(1);
|
||||
expect(onAdmitted).toHaveBeenCalledTimes(otherIssue ? 0 : 1);
|
||||
expect(publishIssueActivity).not.toHaveBeenCalled();
|
||||
expect(deps.executeRun).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(agentWakeupRequests)).toEqual([]);
|
||||
expect(await db.select().from(heartbeatRuns)).toHaveLength(kind === "fresh" ? 0 : 1);
|
||||
});
|
||||
|
||||
it("admits the owner's context compaction maintenance request while its conversation waits", async () => {
|
||||
const f = await waitingCompactionFixture();
|
||||
const accepted = await f.service.request({ companyId: f.company.id, issue: f.issue, userId: f.user.id });
|
||||
@@ -287,8 +341,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake },
|
||||
recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined },
|
||||
});
|
||||
f.deps.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope);
|
||||
f.deps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution);
|
||||
f.deps.wakeQueue.admitIssueWake.mockImplementation(admission.admitIssueWake);
|
||||
let releaseAvailability!: () => void;
|
||||
const bothAvailable = new Promise<void>(resolve => { releaseAvailability = resolve; });
|
||||
let arrivals = 0;
|
||||
@@ -333,7 +386,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
const context = enrichWakeContextSnapshot({ contextSnapshot: { ...options.contextSnapshot },
|
||||
reason: options.reason ?? null, source, triggerDetail, payload: options.payload ?? null });
|
||||
await db.transaction(async tx => {
|
||||
await createWakeAdmissionWriter().insertNewDeferredWake(TransactionScope.create(f.company.id, tx), {
|
||||
await createWakeAdmissionWriter(tx, f.company.id).insertNewDeferredWake({
|
||||
companyId: f.company.id, agentId, source, triggerDetail,
|
||||
payload: { ...options.payload, manualUserWake: true, [DEFERRED_WAKE_CONTEXT_KEY]: context.contextSnapshot },
|
||||
requestedByActorType: options.requestedByActorType ?? null,
|
||||
@@ -553,7 +606,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
}).returning();
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
const store = createFreshIssueRunStore(TransactionScope.create(local.company.id, tx), local.company.id);
|
||||
const store = createFreshIssueRunStore(tx, local.company.id);
|
||||
expect(await store.saveReceipt({
|
||||
kind: "promote", wakeId: localDeferredWake.id, agentId: foreignAgent.id, payload: null,
|
||||
})).toBeUndefined();
|
||||
@@ -889,8 +942,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {
|
||||
mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake },
|
||||
recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined },
|
||||
});
|
||||
deps.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope);
|
||||
deps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution);
|
||||
deps.wakeQueue.admitIssueWake.mockImplementation(admission.admitIssueWake);
|
||||
const queue = createHeartbeatQueue(db, deps);
|
||||
if (scenario === "running") {
|
||||
deps.canStartNewExternalBotWork.mockResolvedValue(true);
|
||||
|
||||
@@ -91,7 +91,6 @@ import {
|
||||
DEFERRED_CHAT_WAKE_RETRY_KEY,
|
||||
assertDurableChatWakeupReceipt,
|
||||
assertDurableChatWakeupRequest,
|
||||
withDurableChatAdmission,
|
||||
authorizeFailedChatRunRetryWake,
|
||||
FailedChatRunRetryAuthorizationError,
|
||||
unadmittedChatWakeupCondition,
|
||||
@@ -219,7 +218,7 @@ import type { budgetService } from "../budgets.js";
|
||||
import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js";
|
||||
import type { instanceSettingsService } from "../instance-settings.js";
|
||||
import type { createRunDispatch } from "../../modules/run-dispatch/index.js";
|
||||
import { bindFreshIssueRunAdmission, hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js";
|
||||
import { crashPoint } from "@paperclipai/shared/crash-points";
|
||||
|
||||
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
|
||||
@@ -247,7 +246,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
|
||||
budgets: Pick<ReturnType<typeof budgetService>, "getInvocationBlock">;
|
||||
instanceSettings: Pick<ReturnType<typeof instanceSettingsService>, "getExperimental">;
|
||||
runDispatch: Pick<ReturnType<typeof createRunDispatch>, "cancelStaleQueuedRun">;
|
||||
wakeQueue: Pick<ReturnType<typeof createWakeQueue>, "createAdmissionTransactionScope" | "admitWakeBehindIssueExecution">;
|
||||
wakeQueue: Pick<ReturnType<typeof createWakeQueue>, "admitIssueWake">;
|
||||
activeRunExecutions: Set<string>;
|
||||
activeRunExecutionPromises: Set<Promise<void>>;
|
||||
activeWakeupPromises: Set<Promise<unknown>>;
|
||||
@@ -3520,21 +3519,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}
|
||||
|
||||
if (issueId) {
|
||||
// Mention-triggered wakes can request input from another agent, but they must
|
||||
// still respect the issue execution lock so a second agent cannot start on the
|
||||
// same issue workspace while the assignee already has a live run.
|
||||
const agentNameKey = normalizeAgentNameKey(agent.name);
|
||||
|
||||
const cancelledRunsToEmit: (typeof heartbeatRuns.$inferSelect)[] = [];
|
||||
const issueActivityPublications: ActivityPublication[] = [];
|
||||
const issuePostCommitActions: IssuePostCommitAction[] = [];
|
||||
|
||||
const outcome = await db
|
||||
.transaction(withDurableChatAdmission(durableRequest, async (tx) => {
|
||||
await tx.execute(
|
||||
sql`select id from issues where id = ${issueId} and company_id = ${agent.companyId} for update`,
|
||||
);
|
||||
|
||||
return wakeQueue.admitIssueWake(
|
||||
{ companyId: agent.companyId, issueId, durableChatRequest: durableRequest },
|
||||
async (tx, { admitWakeBehindIssueExecution, admitFreshIssueRun }) => {
|
||||
if (executionWaitRequestId) {
|
||||
const [pending] = await tx.select().from(agentWakeupRequests).where(and(
|
||||
eq(agentWakeupRequests.id, executionWaitRequestId), eq(agentWakeupRequests.companyId, agent.companyId),
|
||||
@@ -4725,7 +4718,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
resolvedStrategy,
|
||||
hasResolvablePriorSessionWorkspace,
|
||||
},
|
||||
});
|
||||
}, issueActivityPublications);
|
||||
return { kind: "skipped" as const };
|
||||
}
|
||||
}
|
||||
@@ -4797,46 +4790,39 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
return { kind: "coalesced" as const, run: activeExecutionRun };
|
||||
}
|
||||
|
||||
const admissionScope = wakeQueue.createAdmissionTransactionScope(
|
||||
agent.companyId,
|
||||
tx as unknown as Db,
|
||||
);
|
||||
const admission = await wakeQueue.admitWakeBehindIssueExecution(
|
||||
admissionScope,
|
||||
{
|
||||
companyId: agent.companyId,
|
||||
issueId: issue.id,
|
||||
agentId,
|
||||
agentNameKey,
|
||||
issueExecutionAgentNameKey: issue.executionAgentNameKey,
|
||||
activeExecutionRun: {
|
||||
id: activeExecutionRun.id,
|
||||
agentId: activeExecutionRun.agentId,
|
||||
status: activeExecutionRun.status,
|
||||
contextSnapshot: activeExecutionRun.contextSnapshot,
|
||||
wakeupRequestId: activeExecutionRun.wakeupRequestId,
|
||||
},
|
||||
allowRunCoalescing: isConversation(issue) ? false : opts.allowRunCoalescing,
|
||||
durableReceipt: receiptRequest
|
||||
? {
|
||||
id: receiptRequest.id,
|
||||
requestedAt: receiptRequest.requestedAt,
|
||||
}
|
||||
: undefined,
|
||||
reason,
|
||||
liveRunExecutions,
|
||||
wakeCommentId,
|
||||
forceFreshSession:
|
||||
enrichedContextSnapshot.forceFreshSession === true,
|
||||
contextSnapshot: enrichedContextSnapshot,
|
||||
source,
|
||||
triggerDetail,
|
||||
payload,
|
||||
requestedByActorType: opts.requestedByActorType ?? null,
|
||||
requestedByActorId: opts.requestedByActorId ?? null,
|
||||
idempotencyKey: opts.idempotencyKey ?? null,
|
||||
const admission = await admitWakeBehindIssueExecution({
|
||||
companyId: agent.companyId,
|
||||
issueId: issue.id,
|
||||
agentId,
|
||||
agentNameKey,
|
||||
issueExecutionAgentNameKey: issue.executionAgentNameKey,
|
||||
activeExecutionRun: {
|
||||
id: activeExecutionRun.id,
|
||||
agentId: activeExecutionRun.agentId,
|
||||
status: activeExecutionRun.status,
|
||||
contextSnapshot: activeExecutionRun.contextSnapshot,
|
||||
wakeupRequestId: activeExecutionRun.wakeupRequestId,
|
||||
},
|
||||
);
|
||||
allowRunCoalescing: isConversation(issue) ? false : opts.allowRunCoalescing,
|
||||
durableReceipt: receiptRequest
|
||||
? {
|
||||
id: receiptRequest.id,
|
||||
requestedAt: receiptRequest.requestedAt,
|
||||
}
|
||||
: undefined,
|
||||
reason,
|
||||
liveRunExecutions,
|
||||
wakeCommentId,
|
||||
forceFreshSession:
|
||||
enrichedContextSnapshot.forceFreshSession === true,
|
||||
contextSnapshot: enrichedContextSnapshot,
|
||||
source,
|
||||
triggerDetail,
|
||||
payload,
|
||||
requestedByActorType: opts.requestedByActorType ?? null,
|
||||
requestedByActorId: opts.requestedByActorId ?? null,
|
||||
idempotencyKey: opts.idempotencyKey ?? null,
|
||||
});
|
||||
|
||||
if (admission.kind === "coalesced") {
|
||||
return {
|
||||
@@ -5070,7 +5056,6 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}, issueActivityPublications);
|
||||
}
|
||||
|
||||
const admitFreshIssueRun = bindFreshIssueRunAdmission(agent.companyId, tx as unknown as Db);
|
||||
const newRun = await admitFreshIssueRun({
|
||||
receipt: {
|
||||
agentId,
|
||||
@@ -5125,61 +5110,33 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// transitions to "running" — Fix A (lazy locking).
|
||||
|
||||
return { kind: "queued" as const, run: newRun };
|
||||
}))
|
||||
.catch((error) => {
|
||||
if (isExternalChatWaitAuthorizationContention(error))
|
||||
return { kind: "deferred" as const };
|
||||
throw error;
|
||||
});
|
||||
|
||||
// The admission has committed. What follows is dispatch of saved rows.
|
||||
if (outcome.kind !== "deferred" && outcome.kind !== "skipped") admission?.saved();
|
||||
for (const publication of issueActivityPublications) publishActivity(publication);
|
||||
await executeIssuePostCommitActions(db, issuePostCommitActions);
|
||||
|
||||
// Telemetry for the cancelled runs is best-effort background work.
|
||||
// Fire it here and never await it: none of the lifecycle work below,
|
||||
// nor this function's return, depends on it, so a slow telemetry
|
||||
// lookup must not delay them.
|
||||
for (const cancelledRun of cancelledRunsToEmit) {
|
||||
void emitAgentTaskRun(db, cancelledRun);
|
||||
}
|
||||
|
||||
if (outcome.kind === "durable") {
|
||||
return outcome.receipt.runId ? getRun(outcome.receipt.runId) : null;
|
||||
}
|
||||
if (outcome.kind === "deferred" || outcome.kind === "skipped") {
|
||||
return null;
|
||||
}
|
||||
if (outcome.kind === "coalesced") {
|
||||
await attachQueuedSetupRequest(agent);
|
||||
await startNextQueuedRunForAgent(agent.id);
|
||||
return outcome.run;
|
||||
}
|
||||
if (outcome.kind === "replayed") {
|
||||
if (outcome.run.status === "queued") {
|
||||
await attachQueuedSetupRequest(agent);
|
||||
await startNextQueuedRunForAgent(agent.id);
|
||||
}
|
||||
return outcome.run;
|
||||
}
|
||||
|
||||
const newRun = outcome.run;
|
||||
publishLiveEvent({
|
||||
companyId: newRun.companyId,
|
||||
type: "heartbeat.run.queued",
|
||||
payload: {
|
||||
runId: newRun.id,
|
||||
agentId: newRun.agentId,
|
||||
invocationSource: newRun.invocationSource,
|
||||
triggerDetail: newRun.triggerDetail,
|
||||
wakeupRequestId: newRun.wakeupRequestId,
|
||||
},
|
||||
});
|
||||
|
||||
await attachQueuedSetupRequest(agent);
|
||||
await startNextQueuedRunForAgent(agent.id);
|
||||
return newRun;
|
||||
{
|
||||
saved: () => admission?.saved(),
|
||||
publishIssueActivity() {
|
||||
for (const publication of issueActivityPublications) publishActivity(publication);
|
||||
},
|
||||
executeIssueActions: () => executeIssuePostCommitActions(db, issuePostCommitActions),
|
||||
cancelledRuns: cancelledRunsToEmit,
|
||||
emitCancelledRun: (run) => emitAgentTaskRun(db, run),
|
||||
readRun: getRun,
|
||||
publishQueuedRun(run) {
|
||||
publishLiveEvent({
|
||||
companyId: run.companyId,
|
||||
type: "heartbeat.run.queued",
|
||||
payload: {
|
||||
runId: run.id,
|
||||
agentId: run.agentId,
|
||||
invocationSource: run.invocationSource,
|
||||
triggerDetail: run.triggerDetail,
|
||||
wakeupRequestId: run.wakeupRequestId,
|
||||
},
|
||||
});
|
||||
},
|
||||
attachSetupRequest: () => attachQueuedSetupRequest(agent),
|
||||
dispatch: () => startNextQueuedRunForAgent(agent.id),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (durableRequest) throw new Error("chat_inbound_wakeup_binding_denied");
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { resolve } from "node:path";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agents, companies, completionContracts, databaseWorkPending, environmentLeases, reconcileDatabaseWork, heartbeatRunEvents, heartbeatRuns, issues, issueRelations,
|
||||
import { agents, agentWakeupRequests, companies, completionContracts, databaseWorkPending, environmentLeases, reconcileDatabaseWork, heartbeatRunEvents, heartbeatRuns, issues, issueRelations,
|
||||
issueTreeHolds, issueTreeHoldMembers, issueRecoveryActions, nativeRunFinalizations, nativeRunResults, nativeRunnerCheckpoints } from "@paperclipai/db";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import type { NativeExecutionInputV1 } from "@paperclipai/paperclip-runner";
|
||||
@@ -156,11 +156,12 @@ describeEmbeddedPostgres("remote native retry execution admission", () => {
|
||||
const dependencies = createHeartbeatQueueTestDependencies(f.db);
|
||||
dependencies.getAgent.mockResolvedValue(agent);
|
||||
dependencies.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(f.db).getExperimental());
|
||||
dependencies.wakeQueue.admitWakeBehindIssueExecution.mockResolvedValue({ kind: "deferred" });
|
||||
await createHeartbeatQueue(f.db, dependencies).enqueueWakeup(f.agentId, { source: "assignment", reason: "issue_assigned",
|
||||
requestedByActorType: "system", payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId } });
|
||||
expect(await createHeartbeatQueue(f.db, dependencies).enqueueWakeup(f.agentId, { source: "assignment", reason: "issue_assigned",
|
||||
allowRunCoalescing: false, requestedByActorType: "system", payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId } })).toBeNull();
|
||||
expect(dependencies.executeRun).not.toHaveBeenCalled();
|
||||
expect(dependencies.wakeQueue.admitWakeBehindIssueExecution).toHaveBeenCalledOnce();
|
||||
const [wake] = await f.db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId),
|
||||
sql`${agentWakeupRequests.payload}->>'issueId' = ${f.issueId}`));
|
||||
expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null });
|
||||
expect((await f.db.select().from(issues).where(eq(issues.id, f.issueId)))[0])
|
||||
.toMatchObject({ executionRunId: f.runId, checkoutRunId: f.runId });
|
||||
});
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
import { createAdmissionTransactionScope, createWakeAdmissionReader, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { createAdmitWakeBehindIssueExecution } from "../modules/wake-queue/application/use-cases.js";
|
||||
import { filterZombieCoalesceTarget, heartbeatService } from "./heartbeat.js";
|
||||
import { createWakeAdmissionReader, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { heartbeatService } from "./heartbeat.js";
|
||||
import { mergeCoalescedContextSnapshot } from "./heartbeat/run-preparation.js";
|
||||
import { recoveryService } from "./recovery/index.js";
|
||||
import { inspectWorkspaceSource } from "./workspace-source-inspection.js";
|
||||
import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
|
||||
import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wait.js";
|
||||
import { heartbeatRunEvents, agents, approvals, budgetPolicies, costEvents, completionContracts, issueApprovals, issueTreeHolds, issueTreeHoldMembers, companies, createDb, environmentLeases, environments, heartbeatRuns, issueComments, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, agentWakeupRequests, workspaceOperations, plugins } from "@paperclipai/db";
|
||||
import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "./heartbeat/queue.js";
|
||||
import { createHeartbeatQueue } from "./heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "../__tests__/helpers/heartbeat-queue.js";
|
||||
import { recordRequiredWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";
|
||||
import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js";
|
||||
@@ -1425,11 +1424,11 @@ describe("automatic retained workspace recovery", () => {
|
||||
const incoming = { source: "issue.comment", issueId: f.issueId, wakeReason: "issue_commented",
|
||||
wakeCommentId: followup.id, wakeCommentIds: [followup.id] };
|
||||
await db.transaction(async tx => {
|
||||
const scope = createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db);
|
||||
const reader = createWakeAdmissionReader(tx, f.companyId);
|
||||
const actor = { companyId: f.companyId, wakeupRequestId: wake.id, requestedByActorType: "user", requestedByActorId: "operator" };
|
||||
expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, actor)).toBe(true);
|
||||
expect(await createWakeAdmissionReader().matchesActiveWakeActor(scope, { ...actor, requestedByActorId: "other-user" })).toBe(false);
|
||||
await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun(scope, {
|
||||
expect(await reader.matchesActiveWakeActor(actor)).toBe(true);
|
||||
expect(await reader.matchesActiveWakeActor({ ...actor, requestedByActorId: "other-user" })).toBe(false);
|
||||
await createWakeAdmissionWriter(tx, f.companyId).coalesceIntoActiveExecutionRun({
|
||||
companyId: f.companyId, agentId: original.agentId, activeExecutionRunId: queued.id, activeExecutionRunStatus: queued.status,
|
||||
mergedContextSnapshot: mergeCoalescedContextSnapshot((await run(queued.id)).contextSnapshot, incoming,
|
||||
{ preserveExistingInteractionContinuation: true }),
|
||||
@@ -1454,12 +1453,15 @@ describe("automatic retained workspace recovery", () => {
|
||||
await db.update(agentWakeupRequests).set({ runId: null }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id));
|
||||
await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
const reader = createWakeAdmissionReader(), sameActor = vi.spyOn(reader, "matchesActiveWakeActor");
|
||||
queueDeps.wakeQueue.createAdmissionTransactionScope.mockImplementation(createAdmissionTransactionScope);
|
||||
queueDeps.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(createAdmitWakeBehindIssueExecution({
|
||||
reader, writer: createWakeAdmissionWriter(), helpers: { filterZombieCoalesceTarget, mergeCoalescedContextSnapshot,
|
||||
shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake },
|
||||
}));
|
||||
const admitIssueWake = queueDeps.wakeQueue.admitIssueWake.getMockImplementation()!;
|
||||
queueDeps.wakeQueue.admitIssueWake.mockImplementation((target, evaluate, effects) =>
|
||||
admitIssueWake(target, (tx, admission) => evaluate(tx, {
|
||||
...admission,
|
||||
admitWakeBehindIssueExecution(input) {
|
||||
expect(input.durableReceipt).toBeUndefined();
|
||||
return admission.admitWakeBehindIssueExecution(input);
|
||||
},
|
||||
}), effects));
|
||||
queueDeps.liveRunExecutions.has.mockImplementation(id => t.deps.activeRunExecutions.has(id));
|
||||
const stopped = await run(f.runId);
|
||||
if (kind !== "active") {
|
||||
@@ -1511,8 +1513,6 @@ describe("automatic retained workspace recovery", () => {
|
||||
const current = await run(admitted!.id);
|
||||
const [claimedWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, current.wakeupRequestId!));
|
||||
expect(current.contextSnapshot?.wakeCommentIds).toEqual(ids);
|
||||
expect(sameActor).not.toHaveBeenCalled();
|
||||
expect(queueDeps.wakeQueue.admitWakeBehindIssueExecution.mock.calls.every(([, input]) => input.durableReceipt === undefined)).toBe(true);
|
||||
expect(queueDeps.executeRun).not.toHaveBeenCalled();
|
||||
return { ...t, queued: current, wake: claimedWake, followup, expectedIds: ids, latestAuthor: kind === "deferred-different" ? "third-user" : kind === "active" ? "second-user" : "operator" };
|
||||
}
|
||||
@@ -1887,10 +1887,10 @@ describe("automatic retained workspace recovery", () => {
|
||||
}, 30_000);
|
||||
|
||||
it("preserves ordinary no-issue deferred merge semantics", async () => {
|
||||
const f = await seed(), original = await run(f.runId), writer = createWakeAdmissionWriter();
|
||||
const f = await seed(), original = await run(f.runId);
|
||||
const [carrier] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: original.agentId,
|
||||
source: "automation", status: "deferred_issue_execution", payload: { text: "original" }, requestedByActorType: "system" }).returning();
|
||||
await db.transaction(tx => writer.mergeIntoExistingDeferredWake(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), {
|
||||
await db.transaction(tx => createWakeAdmissionWriter(tx, f.companyId).mergeIntoExistingDeferredWake({
|
||||
companyId: f.companyId, existingDeferredWakeId: carrier.id, mergedPayload: { text: "merged" }, nextCoalescedCount: 1,
|
||||
coalescedReceipt: { agentId: original.agentId, source: "automation", triggerDetail: null, reason: null, payload: { text: "saved" }, requestedByActorType: "system", requestedByActorId: null, idempotencyKey: null, runId: null },
|
||||
}));
|
||||
@@ -1901,12 +1901,12 @@ describe("automatic retained workspace recovery", () => {
|
||||
});
|
||||
|
||||
it("rolls back saved primary and carrier overwrite if incoming mixed-author receipt persistence fails", async () => {
|
||||
const f = await seed(), original = await run(f.runId), writer = createWakeAdmissionWriter();
|
||||
const f = await seed(), original = await run(f.runId);
|
||||
const [first] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "operator", body: "First saved input" }).returning();
|
||||
const [second] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "second-user", body: "Second saved input" }).returning();
|
||||
const [carrier] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: original.agentId,
|
||||
source: "automation", status: "deferred_issue_execution", payload: { issueId: f.issueId, commentId: first.id }, requestedByActorType: "user", requestedByActorId: "operator" }).returning();
|
||||
const merge = (tx: Parameters<Parameters<typeof db.transaction>[0]>[0], id?: string) => writer.mergeIntoExistingDeferredWake(createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), {
|
||||
const merge = (tx: Parameters<Parameters<typeof db.transaction>[0]>[0], id?: string) => createWakeAdmissionWriter(tx, f.companyId).mergeIntoExistingDeferredWake({
|
||||
companyId: f.companyId, existingDeferredWakeId: carrier.id, mergedPayload: { issueId: f.issueId, commentId: second.id }, nextCoalescedCount: 1,
|
||||
coalescedReceipt: { ...(id ? { id } : {}), agentId: original.agentId, source: "automation", triggerDetail: null, reason: "issue_commented", payload: { issueId: f.issueId, commentId: second.id, coalescedIntoWakeupRequestId: "untrusted" }, requestedByActorType: "user", requestedByActorId: "second-user", idempotencyKey: null, runId: null },
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user