mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-01 23:43:16 +01:00
fix(server): bound and scope routine projection recovery
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
043fc50acf
commit
46f2c0bb9a
34 files changed
+227
-84
No files matched your search
@@ -7,14 +7,16 @@ scanned every five minutes, even with no routines or only completed history.
|
||||
## Ownership and trigger
|
||||
|
||||
`routine-run-status.ts` owns the projection, custody checks and repair transaction.
|
||||
The worker loads actual mismatches at startup. It accepts issue hints from issue
|
||||
The worker loads actual mismatches in keyset pages of 50 at startup. It accepts issue hints from issue
|
||||
creation, status/origin changes, checkout/release, routine issue association,
|
||||
execution control writes and review state writes. It uses the existing
|
||||
transaction-aware work notification protocol. An outer transaction owns the hint;
|
||||
a savepoint release does not wake the repair worker before commit.
|
||||
|
||||
A standard execution must name the same routine, invocation, company and linked
|
||||
task. Plugin-managed executions may use their declared custom origin. Repair
|
||||
An execution must retain reciprocal task/invocation links in the same company
|
||||
and belong to a routine in that company. This is the existing routine status
|
||||
owner contract. Origin labels can change when a plugin declaration changes or
|
||||
is removed; retained reciprocal links preserve historical execution custody. Repair
|
||||
locks the routine and task and rechecks the linked invocation before writing.
|
||||
It preserves skipped/coalesced history and unrelated dispatch failures. A task
|
||||
that resumes clears an execution-task transient failure through the same owner
|
||||
@@ -33,8 +35,9 @@ history leave no recurring worker SQL. The existing Cloud and heartbeat schedule
|
||||
enablement gates remain in place. Ordinary routine scheduling is unchanged.
|
||||
|
||||
Supported app writers register `notifyRoutineExecutionStatus(tx, issueId)` before
|
||||
their mutation. Common execution writers without an issue hint request debt
|
||||
reconstruction. Backups and independent SQL import/repair tools must restore
|
||||
their mutation. Common execution writers without an issue hint request debt reconstruction only
|
||||
for their recorded company. Projection debt settles even when that company is
|
||||
paused; company activity does not create or cancel this obligation. Backups and independent SQL import/repair tools must restore
|
||||
before startup or call that same transaction contract; process-local hints do
|
||||
not observe independent SQL or another process. No permanent listener or periodic
|
||||
configured-routine scan is added.
|
||||
|
||||
@@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => {
|
||||
const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000);
|
||||
for (const at of [due, earlier]) {
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns)
|
||||
.set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)));
|
||||
.set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId });
|
||||
await settle(); expect(c.nextWakeAt()).toBe(at.getTime());
|
||||
}
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns)
|
||||
.set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)));
|
||||
.set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId });
|
||||
await quiet(c);
|
||||
});
|
||||
it("delivers a reconciled continuation when marking and resolving commit together", async () => {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { subscribeDatabaseWork } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
@@ -888,7 +890,7 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl);
|
||||
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl, { maxConnections: 6 });
|
||||
else {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-branch-containment-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
@@ -1173,15 +1175,15 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {
|
||||
},
|
||||
});
|
||||
|
||||
const routineEvents: string[] = [];
|
||||
const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event));
|
||||
const heartbeat = heartbeatService(db);
|
||||
const run = await heartbeat.wakeup(agentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId },
|
||||
contextSnapshot: { issueId, wakeReason: "issue_assigned" },
|
||||
});
|
||||
|
||||
let run;
|
||||
try {
|
||||
run = await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned",
|
||||
payload: { issueId }, contextSnapshot: { issueId, wakeReason: "issue_assigned" } });
|
||||
} finally { unsubscribeRoutine(); }
|
||||
expect(routineEvents).toContain("intent"); expect(routineEvents).toContain("settled");
|
||||
expect(run).toBeNull();
|
||||
expect(adapterExecute).not.toHaveBeenCalled();
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { writeExecutionControl } from "../services/execution-control-notifications.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
@@ -122,6 +124,56 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
await settled(); await quiet();
|
||||
expect(await saved(run.id)).toEqual(before); expect((await saved(otherRun.id)).status).toBe("issue_created"); expect(errors).toEqual([]);
|
||||
});
|
||||
it.each(["changed", "removed"])("keeps historical custom-origin custody after its plugin declaration is %s", async mode => {
|
||||
const f = await fixture(), pluginKey = `routine-history-${randomUUID()}`, originId = `operation:${randomUUID()}`;
|
||||
const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0",
|
||||
manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test",
|
||||
author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning();
|
||||
const [resource] = await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey,
|
||||
resourceKind: "routine", resourceKey: "history", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }).returning();
|
||||
const run = await dispatch(f);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId);
|
||||
if (mode === "changed") await db.update(pluginManagedResources).set({ defaultsJson: { issueTemplate: { originId: "new-origin" } } }).where(eq(pluginManagedResources.id, resource.id));
|
||||
else await db.delete(pluginManagedResources).where(eq(pluginManagedResources.id, resource.id));
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!));
|
||||
await start(); await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("reconstructs more than one page and does not lose a committed issue hint behind the startup cursor", async () => {
|
||||
const f = await fixture(), ids = Array.from({ length: 53 }, () => randomUUID()).sort();
|
||||
const issueIds = ids.map(() => randomUUID());
|
||||
await db.insert(issues).values(ids.map((id, index) => ({ id: issueIds[index], companyId: f.companyId, title: "Legacy execution",
|
||||
status: index === 0 ? "todo" : "done", originKind: "routine_execution", originRunId: id, originId: f.routine.id })));
|
||||
await db.insert(routineRuns).values(ids.map((id, index) => ({ id, companyId: f.companyId, routineId: f.routine.id,
|
||||
source: "schedule", linkedIssueId: issueIds[index], status: "issue_created" })));
|
||||
const service = routineExecutionStatusService(db), load = service.load;
|
||||
let entered!: () => void, release!: () => void;
|
||||
const enteredGate = new Promise<void>(resolve => { entered = resolve; });
|
||||
const releaseGate = new Promise<void>(resolve => { release = resolve; });
|
||||
const loaded = vi.spyOn(service, "load").mockImplementationOnce(async (scope, afterId) => {
|
||||
const page = await load(scope, afterId); expect(page).toHaveLength(50); entered(); await releaseGate; return page;
|
||||
});
|
||||
const starting = start(service); await enteredGate;
|
||||
await issueService(db).update(issueIds[0]!, { status: "done" }); release(); await starting;
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
await vi.waitFor(async () => expect((await db.select().from(routineRuns)).every(row => row.status === "completed")).toBe(true));
|
||||
expect(loaded.mock.calls.some(([scope, afterId]) => scope === undefined && afterId !== undefined)).toBe(true);
|
||||
expect(loaded.mock.calls.some(([scope]) => scope?.issueId === issueIds[0])).toBe(true);
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("scopes shared execution writes to their company and settles paused-company projection debt", async () => {
|
||||
const f = await fixture(), other = await fixture(), run = await dispatch(f), otherRun = await dispatch(other);
|
||||
const service = routineExecutionStatusService(db), load = vi.spyOn(service, "load");
|
||||
await start(service); await quiet(); load.mockClear();
|
||||
await db.update(companies).set({ status: "paused" }).where(eq(companies.id, f.companyId));
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, otherRun.linkedIssueId!));
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionStatus,
|
||||
tx => tx.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)), { companyId: f.companyId });
|
||||
await projected(run.id, "completed");
|
||||
expect(load.mock.calls.every(([scope]) => scope?.companyId === f.companyId)).toBe(true);
|
||||
expect((await saved(otherRun.id)).status).toBe("issue_created"); await quiet();
|
||||
await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); });
|
||||
await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => {
|
||||
const f = await fixture(), run = await dispatch(f); await start(); await quiet();
|
||||
let release!: () => void, entered!: () => void;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { RoutineExecutionStatusScope } from "../services/routine-execution-status-notifications.js";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
@@ -18,19 +19,19 @@ afterEach(async () => { await coordinator.stop(); vi.useRealTimers(); });
|
||||
it("loads actual debt once and leaves no recurring owner work", async () => {
|
||||
const service = { load: vi.fn(async () => []), settle: vi.fn(async () => true) };
|
||||
await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready;
|
||||
expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined);
|
||||
expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
|
||||
expect(coordinator.nextWakeAt()).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(service.load).toHaveBeenCalledOnce(); expect(service.settle).not.toHaveBeenCalled(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("reconstructs only an invalidated issue and coalesces duplicate requests", async () => {
|
||||
const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) };
|
||||
const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) };
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
|
||||
service.load.mockResolvedValueOnce([entry]);
|
||||
worker.invalidate("issue"); worker.invalidate("issue");
|
||||
worker.invalidate(JSON.stringify({ issueId: "issue" })); worker.invalidate(JSON.stringify({ issueId: "issue" }));
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(service.load).toHaveBeenLastCalledWith("issue"); expect(service.load).toHaveBeenCalledTimes(2);
|
||||
expect(service.load).toHaveBeenLastCalledWith({ issueId: "issue" }, undefined); expect(service.load).toHaveBeenCalledTimes(2);
|
||||
expect(service.settle).toHaveBeenCalledExactlyOnceWith(expect.objectContaining(entry));
|
||||
expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
@@ -50,7 +51,7 @@ it("retains startup and first work during drain without passive owner calls", as
|
||||
canRun = false;
|
||||
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true) };
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
|
||||
worker.invalidate("issue");
|
||||
worker.invalidate(JSON.stringify({ issueId: "issue" }));
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(service.load).not.toHaveBeenCalled(); expect(service.settle).not.toHaveBeenCalled();
|
||||
canRun = true; await vi.advanceTimersByTimeAsync(30_000);
|
||||
@@ -69,11 +70,11 @@ it("keeps a failed owner across drain and retries after resume", async () => {
|
||||
it("retains a notification that arrives while the startup reconstruction is in flight", async () => {
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) };
|
||||
const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) };
|
||||
service.load.mockImplementationOnce(async () => { await gate; return []; });
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun);
|
||||
await vi.waitFor(() => expect(service.load).toHaveBeenCalledOnce());
|
||||
worker.invalidate("issue"); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready;
|
||||
worker.invalidate(JSON.stringify({ issueId: "issue" })); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready;
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
@@ -81,6 +82,26 @@ it("retains a notification that arrives while the startup reconstruction is in f
|
||||
it("cancels retries and subscriptions at shutdown", async () => {
|
||||
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValue(new Error("offline")) };
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
|
||||
await worker.stop(); worker.invalidate("issue"); await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
await worker.stop(); worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
it("reconstructs a backlog in bounded keyset pages while retaining a scoped hint behind the cursor", async () => {
|
||||
const firstPage = Array.from({ length: 50 }, (_, i) => ({ ...entry, id: `run-${String(i).padStart(3, "0")}` }));
|
||||
const late = { ...entry, id: "run-late", issueId: "late-issue" };
|
||||
const service = {
|
||||
load: vi.fn(async (scope?: RoutineExecutionStatusScope, afterId?: string) => scope?.issueId ? [late]
|
||||
: afterId ? [{ ...entry, id: "run-050" }] : firstPage),
|
||||
settle: vi.fn(async () => true),
|
||||
};
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
|
||||
expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
|
||||
worker.invalidate(JSON.stringify({ issueId: "late-issue" }));
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
expect(service.load).toHaveBeenCalledWith(undefined, "run-049");
|
||||
expect(service.load).toHaveBeenCalledWith({ issueId: "late-issue" }, undefined);
|
||||
expect(service.settle).toHaveBeenCalledTimes(52);
|
||||
expect(service.settle).toHaveBeenCalledWith(expect.objectContaining(late));
|
||||
expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
@@ -38,11 +38,10 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${
|
||||
expect((await db.select().from(routineRuns))[0]).toMatchObject({ status: "issue_created", completedAt: null });
|
||||
}
|
||||
});
|
||||
it.each(["active_issue", "wrong_origin", "wrong_routine", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => {
|
||||
it.each(["active_issue", "wrong_origin", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => {
|
||||
const f = await seed();
|
||||
if (mode === "active_issue") await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id));
|
||||
if (mode === "wrong_origin") await db.update(issues).set({ originKind: "manual" }).where(eq(issues.id, f.issue.id));
|
||||
if (mode === "wrong_routine") await db.update(issues).set({ originId: randomUUID() }).where(eq(issues.id, f.issue.id));
|
||||
if (mode === "wrong_run") await db.update(issues).set({ originRunId: randomUUID() }).where(eq(issues.id, f.issue.id));
|
||||
if (mode === "wrong_company") { const other = await seed("done"); await db.update(routineRuns).set({ companyId: other.companyId }).where(eq(routineRuns.id, f.run.id)); }
|
||||
if (mode === "wrong_routine_company") { const other = await seed("done");
|
||||
@@ -55,6 +54,20 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${
|
||||
expect((await db.select().from(routineRuns)).find(row => row.id === f.run.id)).toEqual(before.find(row => row.id === f.run.id));
|
||||
if (mode === "wrong_routine_company") await expect(routineService(db).syncRunStatusForIssue(f.issue.id)).resolves.toBeNull();
|
||||
});
|
||||
it("uses reciprocal durable custody when a retained origin no longer names the current routine declaration", async () => {
|
||||
const f = await seed("done");
|
||||
await db.update(issues).set({ originId: "previous-declaration-origin" }).where(eq(issues.id, f.issue.id));
|
||||
expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 1, settled: 1, deferred: 0 });
|
||||
expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("completed");
|
||||
});
|
||||
it.each(["unknown-status", "other-code"])("preserves a failed invocation with an unrelated transient marker: %s", async mode => {
|
||||
const f = await seed("in_progress");
|
||||
await db.update(routineRuns).set({ status: "failed", failureReason: null,
|
||||
triggerPayload: { transientFailure: { code: mode === "other-code" ? "unrelated_failure" : "execution_issue_status",
|
||||
status: mode === "unknown-status" ? "unrecognized" : "blocked" } } }).where(eq(routineRuns.id, f.run.id));
|
||||
expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 0, settled: 0, deferred: 0 });
|
||||
expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("failed");
|
||||
});
|
||||
it.each(["resume", "retarget", "routine_transfer"])("rechecks %s after candidate selection and before repair", async mode => {
|
||||
const f = await seed(); const transaction = db.transaction.bind(db);
|
||||
vi.spyOn(db, "transaction").mockImplementationOnce(async callback => {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { routineExecutionStatusService } from "../services/routine-run-status.js";
|
||||
import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js";
|
||||
import { skillVersionRetentionService } from "../services/skill-version-retention.js";
|
||||
import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js";
|
||||
@@ -392,6 +393,11 @@ vi.mock("../services/approval-retention.js", async importOriginal => ({
|
||||
approvalRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })),
|
||||
}));
|
||||
|
||||
vi.mock("../services/routine-run-status.js", async importOriginal => ({
|
||||
...await importOriginal<typeof import("../services/routine-run-status.js")>(),
|
||||
routineExecutionStatusService: vi.fn(() => ({ load: vi.fn(async () => []), settle: vi.fn(async () => false) })),
|
||||
}));
|
||||
|
||||
vi.mock("../services/index.js", () => ({
|
||||
backfillLegacyToolOAuthTokens: vi.fn(async () => ({
|
||||
scannedConnections: 0,
|
||||
@@ -1366,6 +1372,27 @@ describe("startServer feedback export wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([{ cloud: false, enabled: true }, { cloud: true, enabled: false }, { cloud: true, enabled: true }])(
|
||||
"registers routine status only for enabled Cloud scheduling ($cloud/$enabled)", async ({ cloud, enabled }) => {
|
||||
if (cloud) vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
|
||||
else vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
const intervalCallbacks: Array<() => void> = [];
|
||||
const intervals = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void) => {
|
||||
intervalCallbacks.push(callback); return 1 as unknown as ReturnType<typeof setInterval>;
|
||||
}) as typeof setInterval);
|
||||
try {
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.routineExecutionStatus)).toHaveLength(cloud && enabled ? 1 : 0);
|
||||
const service = vi.mocked(routineExecutionStatusService).mock.results.at(-1)?.value;
|
||||
if (cloud && enabled) expect(service?.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
|
||||
for (let index = 0; index < 4; index++) for (const callback of intervalCallbacks) callback();
|
||||
await settleStartupHousekeeping();
|
||||
if (cloud && enabled) expect(service?.load).toHaveBeenCalledOnce();
|
||||
} finally { intervals.mockRestore(); vi.unstubAllEnvs(); }
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps routine ticks and setup cleanup active when heartbeat scheduling is suppressed", async () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({
|
||||
heartbeatSchedulerEnabled: true,
|
||||
|
||||
@@ -2,8 +2,10 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js";
|
||||
|
||||
export async function notifyExecutionRunWork(transaction: object, issueId?: string) {
|
||||
await notifyRoutineExecutionStatus(transaction, issueId);
|
||||
export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string };
|
||||
|
||||
export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) {
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
|
||||
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
|
||||
await notifyDeliveryWork(transaction, queue);
|
||||
@@ -14,9 +16,10 @@ export function writeExecutionControl<T>(
|
||||
db: Db | Parameters<Parameters<Db["transaction"]>[0]>[0],
|
||||
queue: DeliveryQueue | readonly DeliveryQueue[],
|
||||
write: (tx: Db) => PromiseLike<T>,
|
||||
scope: ExecutionWorkScope | undefined,
|
||||
): Promise<T> {
|
||||
return db.transaction(async transaction => {
|
||||
await notifyRoutineExecutionStatus(transaction);
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
|
||||
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
|
||||
return write(transaction as unknown as Db);
|
||||
|
||||
@@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl(
|
||||
await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id });
|
||||
await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id });
|
||||
await notifyUnsafeWorkspaceExportRunChange(tx, run);
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: run.companyId });
|
||||
const [updatedRun] = await tx
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { notifyChatResponseAvailability } from "./chat-response-availability.js";
|
||||
@@ -546,7 +547,7 @@ export async function settleUnrecoverableExecutions(
|
||||
companyId: run.companyId, issueId: task.id, runId: run.id,
|
||||
}).limit(1).for("update");
|
||||
if (!hold) return;
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: run.companyId });
|
||||
const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions.";
|
||||
await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled",
|
||||
resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null,
|
||||
@@ -596,6 +597,7 @@ export async function settleUnrecoverableExecutions(
|
||||
if (task.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: task.companyId, issueId: task.id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, task.id, task.companyId);
|
||||
const [projected] = await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
@@ -654,7 +656,7 @@ export async function settleUnrecoverableExecutions(
|
||||
replay: "not_authorized",
|
||||
},
|
||||
});
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: run.companyId });
|
||||
await tx
|
||||
.update(heartbeatRuns)
|
||||
.set({ executionStatusDeliveryId: randomUUID() })
|
||||
|
||||
@@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: {
|
||||
}).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId),
|
||||
eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
if (cancelledStartupIds.size) await db.transaction(async tx => {
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId, agentId });
|
||||
for (const runId of cancelledStartupIds) {
|
||||
await tx.update(nativeRunFinalizations).set({
|
||||
phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null,
|
||||
|
||||
@@ -3724,7 +3724,7 @@ export function heartbeatService(
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx
|
||||
.update(heartbeatRuns)
|
||||
.set({ executionControlDeadlineAt: null })
|
||||
.where(eq(heartbeatRuns.id, run.id)));
|
||||
.where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId });
|
||||
}
|
||||
nativeOwnershipHeld =
|
||||
nativeOwnershipHeld ||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
|
||||
@@ -3919,6 +3920,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
if (issue.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: issue.companyId, issueId: issue.id });
|
||||
await notifyRoutineExecutionStatus(tx, issue.id, issue.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
|
||||
@@ -1044,8 +1044,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
|
||||
if (previous && isHeartbeatRunTerminalStatus(status))
|
||||
await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id });
|
||||
if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId);
|
||||
await notifyExecutionRunWork(tx, previous?.nativeIssueId ?? previous?.issueId
|
||||
?? readNonEmptyString(parseObject(previous?.contextSnapshot).issueId) ?? undefined);
|
||||
await notifyExecutionRunWork(tx, previous ? { companyId: previous.companyId, agentId: previous.agentId, runId: previous.id,
|
||||
issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined } : undefined);
|
||||
if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id,
|
||||
issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined });
|
||||
return write(tx);
|
||||
|
||||
@@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
eq(heartbeatRuns.id, run.id),
|
||||
eq(heartbeatRuns.status, "running"),
|
||||
),
|
||||
));
|
||||
), { companyId: run.companyId });
|
||||
const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded";
|
||||
await recordWorkspaceFinalize(workspaceFinalizeStatus);
|
||||
if (adapterResult.nativeFinalization) {
|
||||
@@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
eq(heartbeatRuns.id, run.id),
|
||||
eq(heartbeatRuns.status, "running"),
|
||||
),
|
||||
));
|
||||
), { companyId: run.companyId });
|
||||
if (
|
||||
issueRef &&
|
||||
context.resumeSessionGoalHeartbeat === true &&
|
||||
|
||||
@@ -392,7 +392,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now })
|
||||
.where(and(eq(issueRecoveryActions.id, existing.id),
|
||||
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
|
||||
sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning());
|
||||
sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId });
|
||||
if (updated) return toReadModel(updated);
|
||||
}
|
||||
return existing;
|
||||
@@ -467,7 +467,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
});
|
||||
}, { companyId: input.companyId });
|
||||
if (!exhausted) {
|
||||
return retryUpsertSourceScoped(input, retryCount);
|
||||
}
|
||||
@@ -539,7 +539,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
});
|
||||
}, { companyId: input.companyId });
|
||||
if (!updated) {
|
||||
return retryUpsertSourceScoped(input, retryCount);
|
||||
}
|
||||
@@ -554,7 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
.insert(issueRecoveryActions)
|
||||
.values(buildInsertValues(input, ownerType, now))
|
||||
.returning();
|
||||
});
|
||||
}, { companyId: input.companyId });
|
||||
return toReadModel(created!);
|
||||
} catch (error) {
|
||||
if (!isUniqueRecoveryActionConflict(error)) throw error;
|
||||
@@ -608,7 +608,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
})
|
||||
.where(and(...predicates))
|
||||
.returning();
|
||||
});
|
||||
}, { companyId: input.companyId });
|
||||
|
||||
return updated ? toReadModel(updated) : null;
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import { notifyNativeReviewResponse } from "./native-review-response-notificatio
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
|
||||
export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) {
|
||||
await notifyRoutineExecutionStatus(tx, scope.issueId);
|
||||
await notifyRoutineExecutionStatus(tx, scope.issueId, scope.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyExternalConversationSettlement(tx, scope);
|
||||
await notifyNativeReviewResponse(tx, scope);
|
||||
|
||||
@@ -10607,7 +10607,7 @@ export function issueService(db: Db) {
|
||||
}
|
||||
if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id);
|
||||
if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id, companyId);
|
||||
const [issue] = await tx.insert(issues).values(values).returning();
|
||||
await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId });
|
||||
if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) {
|
||||
|
||||
@@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: {
|
||||
};
|
||||
if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId });
|
||||
await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id });
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: current.companyId });
|
||||
let [updated] = await tx
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
|
||||
@@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
|
||||
let retained = !retainFiles || alreadyRetained ? run : await recordRequiredWorkspaceRestoreFailure(tx as unknown as Db, run, { workspaceRestoreFailure: "restore_failed" }, source);
|
||||
if (!retained || retainFiles && !hasRequiredWorkspaceRecovery(retained.resultJson)) return;
|
||||
if (options.allowPreterminal && coordinator.phase === "observed") {
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: run.companyId });
|
||||
const [settled] = await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure",
|
||||
leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, nextAttemptAt: null,
|
||||
recoveryState: "blocked", failureCode: coordinator.failureCode ?? "native_continuation_requires_reconciliation", updatedAt: new Date(),
|
||||
|
||||
@@ -899,7 +899,7 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
recycledProviderPids: providerProcesses.recycledPids,
|
||||
stderrTail: row.run.stderrExcerpt,
|
||||
});
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: row.run.companyId });
|
||||
await tx
|
||||
.update(nativeRunFinalizations)
|
||||
.set({
|
||||
|
||||
@@ -401,7 +401,7 @@ async function recordRetryableFailure(input: {
|
||||
if (phase === "terminal_failure" && failureCode === "native_workspace_sync_out_unsafe_archive") {
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId });
|
||||
}
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: input.run.companyId });
|
||||
await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId });
|
||||
await tx
|
||||
.update(nativeRunFinalizations)
|
||||
@@ -1659,7 +1659,7 @@ export async function finalizeNativeRun(input: {
|
||||
if (input.projectRunStatus) await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id });
|
||||
if (input.projectRunStatus) await notifyUnsafeWorkspaceExportRunChange(tx, run);
|
||||
if (input.projectRunStatus) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId);
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: run.companyId });
|
||||
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: coordinator.issueId });
|
||||
const [after] = await projectRun(tx);
|
||||
return after ? { before, after } : null;
|
||||
|
||||
@@ -138,7 +138,7 @@ export async function projectCommittedRun(input: {
|
||||
await notifyCancelledWorkspacePreservation(tx, { companyId: input.run.companyId, runId: input.run.id });
|
||||
await notifyUnsafeWorkspaceExportRunChange(tx, input.run);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.run.companyId);
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: input.run.companyId });
|
||||
await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId });
|
||||
const [after] = await tx
|
||||
.update(heartbeatRuns)
|
||||
|
||||
@@ -267,7 +267,7 @@ export async function reconcileSafeNativeReplacements(
|
||||
current.failureDetail?.replacementDenied
|
||||
)
|
||||
return;
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: run.companyId });
|
||||
await tx
|
||||
.update(nativeRunFinalizations)
|
||||
.set({
|
||||
@@ -409,7 +409,7 @@ export async function reconcileSafeNativeReplacements(
|
||||
idempotencyKey: `native-safe-replacement:${run.id}`,
|
||||
})
|
||||
.returning();
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: run.companyId });
|
||||
await tx.insert(heartbeatRuns).values({
|
||||
id: successorRunId,
|
||||
companyId: run.companyId,
|
||||
|
||||
@@ -9691,7 +9691,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
|
||||
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
|
||||
),
|
||||
));
|
||||
), { companyId: input.execution.binding.companyId });
|
||||
|
||||
const failedAtMs = Date.now();
|
||||
const executionFailureMessage = redactIdentityText(
|
||||
@@ -9743,7 +9743,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
|
||||
eq(nativeRunFinalizations.attempt, attempt),
|
||||
isNull(nativeRunFinalizations.resultId),
|
||||
)).returning({ runId: nativeRunFinalizations.runId }));
|
||||
)).returning({ runId: nativeRunFinalizations.runId }), { companyId: input.execution.binding.companyId });
|
||||
if (settled) {
|
||||
await stoppedLeaseRenewal;
|
||||
if (warmSessionId !== null && lifecyclePolicy.mode === "warm") {
|
||||
@@ -9830,7 +9830,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
),
|
||||
)
|
||||
.for("update");
|
||||
await notifyExecutionRunWork(tx);
|
||||
await notifyExecutionRunWork(tx, { companyId: input.execution.binding.companyId });
|
||||
await notifyNativeFinalizationWork(tx, input.execution.binding);
|
||||
const updated = await tx
|
||||
.update(nativeRunFinalizations)
|
||||
|
||||
@@ -39,7 +39,7 @@ export async function retireSupersededNativeRetry(db: Db, input: {
|
||||
)).limit(1);
|
||||
if (!source) return false;
|
||||
|
||||
await notifyExecutionRunWork(db);
|
||||
await notifyExecutionRunWork(db, { companyId: run.companyId });
|
||||
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", nextAttemptAt: null,
|
||||
leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: "blocked",
|
||||
failureCode: "native_retry_superseded", failureDetail: { ...coordinator.failureDetail,
|
||||
|
||||
@@ -38,7 +38,7 @@ describe("reconstruction after confirmed native workspace loss", () => {
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_RECONSTRUCTION_TEST_DATABASE_URL");
|
||||
if (external) {
|
||||
await applyPendingMigrations(external);
|
||||
db = createDb(external);
|
||||
db = createDb(external, { maxConnections: 6 });
|
||||
return;
|
||||
}
|
||||
temporary = await startEmbeddedPostgresTestDatabase("workspace-reconstruction-");
|
||||
@@ -144,17 +144,21 @@ describe("reconstruction after confirmed native workspace loss", () => {
|
||||
expect(await queue.nextTimerHeartbeatAt(agent)).toBeNull();
|
||||
const events: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.agentTimers, event => events.push(event));
|
||||
const routineEvents: string[] = [];
|
||||
const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event));
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
await reconstructNativeWorkspace({ ...f, db: tx as unknown as Db });
|
||||
expect(events).toContain("intent");
|
||||
expect(events).not.toContain("settled");
|
||||
expect(routineEvents).toContain("intent"); expect(routineEvents).not.toContain("settled");
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("blocked");
|
||||
});
|
||||
expect(events).toContain("settled");
|
||||
expect(routineEvents).toContain("settled");
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("todo");
|
||||
expect(await queue.nextTimerHeartbeatAt(agent)).toBe(agent.createdAt.getTime() + 600_000);
|
||||
} finally { unsubscribe(); }
|
||||
} finally { unsubscribe(); unsubscribeRoutine(); }
|
||||
});
|
||||
|
||||
it("pins a historical cancelled workspace through public heartbeat startup without replaying its turn", async () => {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
|
||||
import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm";
|
||||
@@ -187,6 +188,7 @@ export async function reconstructNativeWorkspace(input: {
|
||||
}).where(eq(issueRecoveryActions.id, source.action.id));
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, input.issueId, input.companyId);
|
||||
await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null,
|
||||
blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id));
|
||||
const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId,
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export function notifyRoutineExecutionStatus(tx: object, issueId?: string) {
|
||||
return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus, issueId ?? "*");
|
||||
export type RoutineExecutionStatusScope = { issueId?: string; companyId?: string };
|
||||
|
||||
export function notifyRoutineExecutionStatus(tx: object, issueId?: string, companyId?: string) {
|
||||
return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus,
|
||||
issueId || companyId ? JSON.stringify({ issueId, companyId } satisfies RoutineExecutionStatusScope) : "*");
|
||||
}
|
||||
@@ -1,18 +1,27 @@
|
||||
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
|
||||
import type { routineExecutionStatusService } from "./routine-run-status.js";
|
||||
import type { RoutineExecutionStatusScope } from "./routine-execution-status-notifications.js";
|
||||
import { ROUTINE_EXECUTION_STATUS_PAGE_SIZE, type routineExecutionStatusService } from "./routine-run-status.js";
|
||||
|
||||
export function registerRoutineExecutionStatusWork(
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
service: ReturnType<typeof routineExecutionStatusService>,
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
const cursors = new Map<string, string>();
|
||||
return registerScopedDeadlineWork(coordinator, {
|
||||
queue: DELIVERY_QUEUES.routineExecutionStatus,
|
||||
decodeScope: hint => hint === "*" ? undefined : hint,
|
||||
matches: (entry, issueId?: string) => issueId === undefined || entry.issueId === issueId,
|
||||
load: async (issueId?: string) => (await service.load(issueId)).map(row => ({ ...row, dueAt: Date.now() })),
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RoutineExecutionStatusScope,
|
||||
matches: (entry, scope?: RoutineExecutionStatusScope) => (!scope?.issueId || entry.issueId === scope.issueId)
|
||||
&& (!scope?.companyId || entry.companyId === scope.companyId),
|
||||
load: async (scope?: RoutineExecutionStatusScope) => {
|
||||
const key = JSON.stringify(scope) ?? "*";
|
||||
const rows = await service.load(scope, cursors.get(key));
|
||||
const complete = rows.length < ROUTINE_EXECUTION_STATUS_PAGE_SIZE;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
|
||||
},
|
||||
check: async entry => { await service.settle(entry); return null; },
|
||||
}, canRun);
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyRoutineExecutionStatus, type RoutineExecutionStatusScope } from "./routine-execution-status-notifications.js";
|
||||
import { and, eq, inArray, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { issues, pluginManagedResources, routineRuns, routines } from "@paperclipai/db";
|
||||
import { issues, routineRuns, routines } from "@paperclipai/db";
|
||||
|
||||
const EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE = "execution_issue_status";
|
||||
const EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES = ["blocked", "cancelled"] as const;
|
||||
@@ -141,17 +141,11 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string
|
||||
}
|
||||
|
||||
|
||||
export function routineExecutionOriginCondition() {
|
||||
return or(sql`${issues.originId} = ${routineRuns.routineId}::text`, sql`exists (
|
||||
select 1 from ${pluginManagedResources} resource
|
||||
where resource.company_id = ${routineRuns.companyId}
|
||||
and resource.resource_kind = 'routine' and resource.resource_id = ${routineRuns.routineId}
|
||||
and resource.defaults_json -> 'issueTemplate' ->> 'originId' = ${issues.originId}
|
||||
)`);
|
||||
}
|
||||
export const ROUTINE_EXECUTION_STATUS_PAGE_SIZE = 50;
|
||||
|
||||
export function routineExecutionStatusDebtCondition() {
|
||||
const transientFailure = sql`(${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE}
|
||||
const transientFailure = sql`((${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE}
|
||||
and ${routineRuns.triggerPayload} -> 'transientFailure' ->> 'status' in ('blocked', 'cancelled'))
|
||||
or ${routineRuns.failureReason} in ('Execution issue moved to blocked', 'Execution issue moved to cancelled'))`;
|
||||
return or(
|
||||
and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "blocked", "cancelled"])),
|
||||
@@ -165,15 +159,18 @@ export function routineExecutionStatusDebtCondition() {
|
||||
}
|
||||
|
||||
export function routineExecutionStatusService(db: Db) {
|
||||
async function load(issueId?: string) {
|
||||
async function load(scope?: RoutineExecutionStatusScope, afterId?: string) {
|
||||
return db.select({ id: routineRuns.id, companyId: routineRuns.companyId,
|
||||
issueId: issues.id, routineId: routineRuns.routineId })
|
||||
.from(routineRuns)
|
||||
.innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId)))
|
||||
.innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), eq(issues.companyId, routineRuns.companyId),
|
||||
eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`, routineExecutionOriginCondition()))
|
||||
.where(and(routineExecutionStatusDebtCondition(), ...(issueId ? [eq(issues.id, issueId)] : [])))
|
||||
.orderBy(routineRuns.id);
|
||||
eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`))
|
||||
.where(and(routineExecutionStatusDebtCondition(),
|
||||
...(scope?.issueId ? [eq(issues.id, scope.issueId)] : []),
|
||||
...(scope?.companyId ? [eq(routineRuns.companyId, scope.companyId)] : []),
|
||||
...(afterId ? [sql`${routineRuns.id} > ${afterId}::uuid`] : [])))
|
||||
.orderBy(routineRuns.id).limit(ROUTINE_EXECUTION_STATUS_PAGE_SIZE);
|
||||
}
|
||||
async function settle(candidate: Awaited<ReturnType<typeof load>>[number]) {
|
||||
return db.transaction(async tx => {
|
||||
@@ -190,7 +187,7 @@ export function routineExecutionStatusService(db: Db) {
|
||||
.innerJoin(issues, eq(issues.id, routineRuns.linkedIssueId))
|
||||
.where(and(eq(routineRuns.id, candidate.id), eq(routineRuns.companyId, candidate.companyId),
|
||||
eq(routineRuns.routineId, candidate.routineId), eq(routineRuns.linkedIssueId, issue.id),
|
||||
routineExecutionOriginCondition(), routineExecutionStatusDebtCondition())).for("update", { of: routineRuns });
|
||||
routineExecutionStatusDebtCondition())).for("update", { of: routineRuns });
|
||||
if (!run) return false;
|
||||
return (await syncRoutineRunStatusForIssue(tx as unknown as Db, issue.id)) !== null;
|
||||
});
|
||||
|
||||
@@ -2002,6 +2002,8 @@ export function routineService(
|
||||
}
|
||||
|
||||
|
||||
let statusReconcileCursor: string | undefined;
|
||||
|
||||
async function syncRunStatusForIssue(issueId: string, executor: Db = db) {
|
||||
return syncRoutineRunStatusForIssue(executor, issueId);
|
||||
}
|
||||
@@ -3246,7 +3248,8 @@ export function routineService(
|
||||
|
||||
reconcileExecutionRunStatuses: async (): Promise<{ scanned: number; settled: number; deferred: number }> => {
|
||||
const service = routineExecutionStatusService(db);
|
||||
const candidates = await service.load();
|
||||
const candidates = await service.load(undefined, statusReconcileCursor);
|
||||
statusReconcileCursor = candidates.length === 50 ? candidates.at(-1)!.id : undefined;
|
||||
let settled = 0, deferred = 0;
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
|
||||
@@ -71,7 +71,7 @@ export async function updateSystemRecovery(db: Db, source: WorkspaceRecoverySour
|
||||
sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,revision}' = ${String(previous.revision)}`))
|
||||
.returning({ id: issueRecoveryActions.id });
|
||||
return Boolean(updated);
|
||||
});
|
||||
}, { companyId: source.companyId });
|
||||
}
|
||||
|
||||
export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) {
|
||||
|
||||
@@ -334,7 +334,7 @@ export async function requestTaskRetry(input: {
|
||||
details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } });
|
||||
return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id,
|
||||
chatActionId: intent.chatActionId };
|
||||
});
|
||||
}, { companyId: input.companyId });
|
||||
}
|
||||
|
||||
/** Chat owns admission and authorization. The task receipt follows that
|
||||
|
||||
Reference in new issue
Block a user