mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-08 01:57:31 +01:00
merge: preserve status-card and recovery producers in the review stack
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
commit
9b4beff72b
126 files changed
+4453
-581
No files matched your search
@@ -0,0 +1,109 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, projects, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
vi.mock("../services/activity-log.js", () => ({ logActivity: vi.fn() }));
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("budget history for deleted scopes", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-budget-history-");
|
||||
db = createDb(database.connectionString);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterEach(async () => {
|
||||
await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents);
|
||||
await db.delete(projects); await db.delete(agents); await db.delete(companies);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "History company", issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Current agent", role: "engineer" });
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 300 }).returning();
|
||||
await db.insert(costEvents).values({ companyId, agentId, provider: "openai", model: "history-fixture", inputTokens: 1, outputTokens: 1, costCents: 17, occurredAt: new Date() });
|
||||
return { companyId, agentId, policy: policy! };
|
||||
}
|
||||
|
||||
it.each(["agent", "project"] as const)("keeps current budgets and deleted %s policy/incident history readable without changing stored rows", async scopeType => {
|
||||
const { companyId, policy } = await fixture(), deletedId = randomUUID();
|
||||
if (scopeType === "agent") await db.insert(agents).values({ id: deletedId, companyId, name: "Former agent", role: "engineer" });
|
||||
else await db.insert(projects).values({ id: deletedId, companyId, name: "Former project" });
|
||||
const [historical] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: deletedId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 200 }).returning();
|
||||
await db.insert(budgetIncidents).values({ companyId, policyId: historical!.id, scopeType, scopeId: deletedId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 200, amountObserved: 201 });
|
||||
if (scopeType === "agent") await db.delete(agents).where(eq(agents.id, deletedId));
|
||||
else await db.delete(projects).where(eq(projects.id, deletedId));
|
||||
const policiesBefore = await db.select().from(budgetPolicies), incidentsBefore = await db.select().from(budgetIncidents), costsBefore = await db.select().from(costEvents);
|
||||
|
||||
const overview = await budgetService(db).overview(companyId);
|
||||
expect(overview.policies).toHaveLength(2);
|
||||
expect(overview.policies.find(row => row.policyId === policy.id)).toMatchObject({ companyId, scopeName: "Current agent", amount: 300, observedAmount: 17 });
|
||||
expect(overview.policies.find(row => row.policyId === historical!.id)).toMatchObject({ companyId, scopeType, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, amount: 200 });
|
||||
expect(overview.activeIncidents).toMatchObject([{ companyId, policyId: historical!.id, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, status: "open", amountObserved: 201 }]);
|
||||
expect(await db.select().from(budgetPolicies)).toEqual(policiesBefore);
|
||||
expect(await db.select().from(budgetIncidents)).toEqual(incidentsBefore);
|
||||
expect(await db.select().from(costEvents)).toEqual(costsBefore);
|
||||
});
|
||||
it("rejects missing-scope budget grants but can dismiss their historical incidents", async () => {
|
||||
const { companyId, agentId } = await fixture(), missingId = randomUUID(), service = budgetService(db);
|
||||
await expect(service.evaluateAdmission(companyId, missingId)).rejects.toThrow("Agent not found");
|
||||
await expect(service.evaluateAdmission(companyId, agentId, { projectId: missingId })).rejects.toThrow("Project not found");
|
||||
for (const scopeType of ["agent", "project"] as const) {
|
||||
await expect(service.upsertPolicy(companyId, { scopeType, scopeId: missingId, amount: 100 }, "board")).rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: missingId,
|
||||
windowKind: "calendar_month_utc", amount: 100 }).returning();
|
||||
const [approval] = await db.insert(approvals).values({ companyId, type: "budget_override_required", payload: {} }).returning();
|
||||
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, approvalId: approval!.id, scopeType, scopeId: missingId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 100, amountObserved: 101 }).returning();
|
||||
const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) };
|
||||
await expect(service.resolveIncident(companyId, incident!.id, { action: "raise_budget_and_resume", amount: 200 }, "board"))
|
||||
.rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
|
||||
expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }).toEqual(before);
|
||||
const dismissed = await service.resolveIncident(companyId, incident!.id, { action: "keep_paused" }, "board");
|
||||
expect(dismissed).toMatchObject({ companyId, policyId: policy!.id, scopeId: missingId, scopeName: `Deleted ${scopeType}`, status: "dismissed" });
|
||||
const { status: _status, resolvedAt: _resolvedAt, updatedAt: _updatedAt, ...preserved } = incident!;
|
||||
const [saved] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident!.id));
|
||||
expect(saved).toMatchObject({ ...preserved, status: "dismissed", resolvedAt: expect.any(Date) });
|
||||
expect((await db.select().from(approvals).where(eq(approvals.id, approval!.id)))[0]).toMatchObject({ status: "rejected" });
|
||||
expect(await db.select().from(budgetPolicies)).toEqual(before.policies);
|
||||
expect(await db.select().from(agents)).toEqual(before.agents);
|
||||
expect(await db.select().from(costEvents)).toEqual(before.costs);
|
||||
expect((await service.overview(companyId)).activeIncidents.some(row => row.id === incident!.id)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a corrupted policy pointing to another company's scope instead of exposing its name", async () => {
|
||||
const { companyId } = await fixture(), foreignCompanyId = randomUUID(), foreignAgentId = randomUUID();
|
||||
await db.insert(companies).values({ id: foreignCompanyId, name: "Other company", issuePrefix: `F${foreignCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: foreignAgentId, companyId: foreignCompanyId, name: "Private other-company agent", role: "engineer" });
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: foreignAgentId, windowKind: "calendar_month_utc", amount: 200 }).returning();
|
||||
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, scopeType: "agent", scopeId: foreignAgentId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 200, amountObserved: 201 }).returning();
|
||||
const service = budgetService(db);
|
||||
await expect(service.overview(companyId)).rejects.toThrow("Budget scope not found");
|
||||
for (const scopeId of [foreignAgentId, randomUUID()]) {
|
||||
await db.update(budgetIncidents).set({ scopeId }).where(eq(budgetIncidents.id, incident!.id));
|
||||
const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents) };
|
||||
for (const action of ["raise_budget_and_resume", "keep_paused"] as const) {
|
||||
await expect(service.resolveIncident(companyId, incident!.id, { action, amount: 300 }, "board")).rejects.toThrow("Budget scope not found");
|
||||
expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents) }).toEqual(before);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
@@ -4168,6 +4168,7 @@ describe("company portability", () => {
|
||||
const insertedRelationValues: Array<Record<string, unknown>> = [];
|
||||
const monitorUpdates: Array<Record<string, unknown>> = [];
|
||||
const db = {
|
||||
transaction: async (write: (tx: unknown) => Promise<unknown>) => write(db),
|
||||
insert: () => ({
|
||||
values: (rows: Array<Record<string, unknown>>) => ({
|
||||
onConflictDoNothing: async () => {
|
||||
|
||||
@@ -0,0 +1,427 @@
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
|
||||
import { accessService } from "../services/access.js";
|
||||
import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js";
|
||||
import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js";
|
||||
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
|
||||
import { issueTreeControlService } from "../services/issue-tree-control.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyNativeStatusWakeWork, notifyTaskRetrySettlement } from "../services/deferred-execution-notifications.js";
|
||||
import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-receipt-work.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { requestTaskRetry } from "../services/task-retry.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
describe("deferred receipt durable dispatch", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
let drain = false;
|
||||
const errors: unknown[] = [];
|
||||
const passes = new Set<Promise<unknown>>();
|
||||
beforeAll(async () => {
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL");
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0;
|
||||
heartbeat = heartbeatService(db);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const pass = Promise.resolve(task.run(signal)); passes.add(pass);
|
||||
void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass;
|
||||
} });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
|
||||
vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), actorId = "deferred-receipt-user";
|
||||
await db.insert(authUsers).values({ id: actorId, name: "Receipt operator", email: `${actorId}@example.test`,
|
||||
createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companies).values({ id: companyId, name: "Deferred dispatch", issuePrefix: `DE${companyId.slice(0, 6)}`,
|
||||
defaultResponsibleUserId: actorId, requireBoardApprovalForNewAgents: false });
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: actorId, principalType: "user", status: "active", membershipRole: "owner" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Receipt process", adapterType: "process", status: "idle",
|
||||
adapterConfig: { command: process.execPath, args: ["-e", "console.log('receipt-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Resume saved task", status: "todo", assigneeAgentId: agent.id }).returning();
|
||||
const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, issueId: issue.id,
|
||||
status: "failed", finishedAt: new Date(), responsibleUserId: actorId, errorCode: "process_failed",
|
||||
contextSnapshot: { issueId: issue.id }, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
|
||||
return { companyId, agent, issue, source, actorId };
|
||||
}
|
||||
async function quiet() {
|
||||
await vi.waitFor(async () => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
|
||||
const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute");
|
||||
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed);
|
||||
}
|
||||
async function finish(issueId: string) {
|
||||
await issueService(db).update(issueId, { status: "done" });
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
|
||||
}, { timeout: 15_000 });
|
||||
const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"));
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
if (saved.length) {
|
||||
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
|
||||
while (Date.now() <= due) {
|
||||
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
|
||||
await Promise.allSettled([...passes]); await immediate();
|
||||
}
|
||||
}
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(500); await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0);
|
||||
});
|
||||
}
|
||||
it("is quiet after empty reconstruction and terminal receipt history", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet();
|
||||
});
|
||||
it("dispatches an ordinary committed task Retry into saved process output", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
expect(receipt.accepted).toBe(true);
|
||||
let runId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId!;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
await finish(f.issue.id);
|
||||
const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`));
|
||||
expect(linked).toHaveLength(2);
|
||||
expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId]));
|
||||
expect(linked.map(row => row.idempotencyKey)).toContain(`task-retry-dispatch:${receipt.requestId}`);
|
||||
expect(linked.find(row => row.id === receipt.requestId)?.status).toBe("coalesced");
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1);
|
||||
await quiet();
|
||||
});
|
||||
it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => {
|
||||
const f = await fixture(), updatedAt = new Date(), native = kind === "native claim";
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "automation", status: native ? "claimed" : "deferred_issue_execution", claimedAt: updatedAt, updatedAt,
|
||||
payload: native ? {} : { executionWait: { reason: "recovery" } },
|
||||
requestedByActorType: "system", requestedByActorId: native ? "native-status-committer" : null }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const check = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
|
||||
await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, scope.requestId!));
|
||||
});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: check, settleTaskChat: vi.fn(), dispatchNative: check,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const due = updatedAt.getTime() + (native ? 60_000 : 30_000);
|
||||
expect(coordinator.nextWakeAt()).toBe(due); expect(check).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(check).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await vi.waitFor(() => expect(check).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id })));
|
||||
await quiet();
|
||||
});
|
||||
it("reconciles the first drained producer and waits for resume without domain SQL", async () => {
|
||||
const f = await fixture(); drain = true;
|
||||
const resume = vi.fn(async () => {}), dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await db.transaction(async tx => {
|
||||
await notifyDeferredExecutionWork(tx, { companyId: f.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, source: "on_demand",
|
||||
status: "deferred_issue_execution", reason: "task_retry" });
|
||||
});
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(resume).not.toHaveBeenCalled(); expect(dispatchNative).not.toHaveBeenCalled();
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
drain = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(1)); await quiet();
|
||||
});
|
||||
it.each([
|
||||
{ name: "saved comment", payload: { _paperclipWakeContext: { wakeCommentIds: ["saved-comment"] } } },
|
||||
{ name: "interaction", payload: { mutation: "interaction" } },
|
||||
{ name: "execution wait", payload: { executionWait: { reason: "recovery" } } },
|
||||
{ name: "interrupt", payload: { queuedCommentInterrupt: { actorId: "operator" } } },
|
||||
])("activates only the committed company's $name receipt after an empty start", async ({ payload }) => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
const resume = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const [other] = await db.insert(agentWakeupRequests).values({ companyId: foreign.companyId, agentId: foreign.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload }).returning();
|
||||
let requestId: string;
|
||||
await db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await notifyDeferredExecutionWork(nested, { companyId: f.companyId });
|
||||
const [row] = await nested.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: new Date(Date.now() - 31_000) }).returning();
|
||||
requestId = row.id;
|
||||
});
|
||||
expect(resume).not.toHaveBeenCalled();
|
||||
});
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId })), { timeout: 5_000 });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, other.id)))[0].status).toBe("deferred_issue_execution");
|
||||
await quiet();
|
||||
});
|
||||
it("reconstructs a cancelled legacy run's exact deferred queue even without comment payload", async () => {
|
||||
const f = await fixture();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution" }).returning();
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { queuedCommentInterruptQueueId: wake.id } })
|
||||
.where(eq(heartbeatRuns.id, f.source.id));
|
||||
const resume = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id }));
|
||||
await quiet();
|
||||
});
|
||||
it("settles a processed chat receipt without restaging it until an actual eligibility commit", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Receipt chat", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Receipt chat", transport: "rest_api" }).returning();
|
||||
const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection.id,
|
||||
provider: "slack", publicId: randomUUID(), assignedAgentId: f.agent.id }).returning();
|
||||
const intentId = randomUUID(), actionId = randomUUID(), previous = new Date(Date.now() - 60_000);
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
{ id: intentId, companyId: f.companyId, agentId: f.agent.id, source: "on_demand", reason: "task_retry",
|
||||
status: "deferred_issue_execution", updatedAt: previous, payload: { issueId: f.issue.id,
|
||||
taskRetry: { schema: "paperclip.task-retry.v1", generationId: intentId, sourceRunId: f.source.id,
|
||||
actorId: f.actorId, strategy: "evaluate", chatActionId: actionId } } },
|
||||
{ id: actionId, companyId: f.companyId, agentId: f.agent.id, source: "automation", status: "deferred_issue_execution",
|
||||
payload: { issueId: f.issue.id, taskRetryRequestId: intentId } },
|
||||
]);
|
||||
await db.transaction(async tx => {
|
||||
await notifyTaskRetrySettlement(tx, { companyId: f.companyId, requestId: intentId });
|
||||
await tx.insert(chatActions).values({ id: actionId, companyId: f.companyId, endpointId: endpoint.id,
|
||||
providerActionId: randomUUID(), kind: "failed_run_retry", status: "processed", updatedAt: new Date(),
|
||||
payload: { issueId: f.issue.id, taskRetryIntentId: intentId, failedRunId: f.source.id } });
|
||||
});
|
||||
await vi.waitFor(async () => {
|
||||
const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, intentId));
|
||||
expect(intent.updatedAt.getTime()).toBeGreaterThan(previous.getTime());
|
||||
}, { timeout: 5_000 });
|
||||
expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status).toBe("processed");
|
||||
await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
|
||||
await vi.waitFor(async () => expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status)
|
||||
.toBe("issued"), { timeout: 5_000 });
|
||||
await quiet();
|
||||
});
|
||||
it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => {
|
||||
const f = await fixture(), initialAt = new Date();
|
||||
const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({
|
||||
companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId,
|
||||
}))).returning();
|
||||
const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId },
|
||||
_paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } };
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: initialAt }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps);
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: scope => queue.resumeDeferredExecutionReceipts(scope),
|
||||
settleTaskChat: async () => {}, dispatchNative: async () => {}, schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
|
||||
const movedAt = new Date();
|
||||
const writer = createQueuedCommentIssueLockWriter(db, { syncCommentReferences: async () => {},
|
||||
deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {} });
|
||||
await writer.withLockedQueue({ issue: f.issue, queueId: wake.id,
|
||||
actor: { actorType: "user", actorId: f.actorId, agentId: null, runId: null, agentApiKeyId: null } },
|
||||
async (_locked, tx) => { await tx.updateWakeQueuedCommentIds({ wakeId: wake.id, payload,
|
||||
ids: comments.map(row => row.id).reverse(), updatedAt: movedAt }); });
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
expect(deps.resumeQueuedCommentInterrupt).not.toHaveBeenCalled();
|
||||
expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(deps.resumeQueuedCommentInterrupt).toHaveBeenCalledExactlyOnceWith(f.companyId, wake.id));
|
||||
await quiet();
|
||||
});
|
||||
it("reactivates a held task Retry when the authorized subtree hold is released without wakeAgents", async () => {
|
||||
const f = await fixture(), tree = issueTreeControlService(db);
|
||||
const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
|
||||
const hold = await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
expect(request.requestId).toBeTruthy();
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.payload?.executionWait).toMatchObject({ reason: "issue_tree_hold_active" });
|
||||
expect(wake.runId).toBeNull();
|
||||
}, { timeout: 5_000 });
|
||||
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
|
||||
await tree.releaseHold(f.companyId, f.issue.id, hold.hold.id, { actor, reason: "Resume saved Retry" });
|
||||
let runId: string | null = null;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
|
||||
}, { timeout: 40_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id); await quiet();
|
||||
});
|
||||
it("keeps a moved native dispatch error deadline instead of retrying at its old time", async () => {
|
||||
const f = await fixture(), initialAt = new Date();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "automation", status: "queued", error: "Transient dispatch failure", updatedAt: initialAt,
|
||||
requestedByActorType: "system", requestedByActorId: "native-status-committer" }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: async () => {}, settleTaskChat: async () => {},
|
||||
dispatchNative, schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
|
||||
const movedAt = new Date();
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: movedAt }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
expect(dispatchNative).not.toHaveBeenCalled(); expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(dispatchNative).toHaveBeenCalledTimes(1)); await quiet();
|
||||
});
|
||||
it("arms one new wait reason grace and stays quiet when that same gate is still closed", async () => {
|
||||
const f = await fixture(); let reason = "operator_pause";
|
||||
const resume = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
|
||||
await writeDeferredExecutionWait(db, { companyId: scope.companyId!, requestId: scope.requestId! }, { reason });
|
||||
});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {},
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", reason: "task_retry" }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
for (const nextReason of ["operator_pause", "budget_pause"]) {
|
||||
reason = nextReason;
|
||||
await writeDeferredExecutionWait(db, { companyId: f.companyId, requestId: wake.id }, { reason });
|
||||
const [current] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
const due = current.updatedAt.getTime() + 30_000;
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due));
|
||||
const checked = resume.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now());
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(checked + 1));
|
||||
await quiet();
|
||||
}
|
||||
expect(resume).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("reactivates a saved Retry through ordinary restore cancelled task", async () => {
|
||||
const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
const tree = issueTreeControlService(db);
|
||||
const cancel = await tree.createHold(f.companyId, f.issue.id, { mode: "cancel", actor });
|
||||
await tree.cancelIssueStatusesForHold(f.companyId, f.issue.id, cancel.hold.id);
|
||||
const restore = await tree.createHold(f.companyId, f.issue.id, { mode: "restore", actor });
|
||||
const commitEligibility = () => tree.restoreIssueStatusesForHold(f.companyId, f.issue.id, restore.hold.id, { actor });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull();
|
||||
await commitEligibility();
|
||||
let runId: string | null = null;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id); await quiet();
|
||||
});
|
||||
it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => {
|
||||
const f = await fixture(), departing = `departing-${randomUUID()}`;
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
await db.insert(authUsers).values({ id: departing, name: "Departing member", email: `${departing}@example.test`,
|
||||
createdAt: new Date(), updatedAt: new Date() });
|
||||
const [member] = await db.insert(companyMemberships).values({ companyId: f.companyId, principalId: departing,
|
||||
principalType: "user", status: "active", membershipRole: "member" }).returning();
|
||||
await issueService(db).update(f.issue.id, { assigneeUserId: departing, assigneeAgentId: null });
|
||||
const readIntent = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0];
|
||||
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
|
||||
const deferred = vi.fn(), native = vi.fn();
|
||||
const unsubscribes = [subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred),
|
||||
subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents, native)];
|
||||
try {
|
||||
await expect(db.transaction(async tx => {
|
||||
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
|
||||
{ reassignment: { assigneeAgentId: f.agent.id } });
|
||||
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
|
||||
throw new Error("roll back member archive");
|
||||
})).rejects.toThrow("roll back member archive");
|
||||
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].assigneeUserId).toBe(departing);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.deferredExecutionReceipts);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents);
|
||||
deferred.mockClear(); native.mockClear();
|
||||
await db.transaction(async tx => {
|
||||
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
|
||||
{ reassignment: { assigneeAgentId: f.agent.id } });
|
||||
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
|
||||
});
|
||||
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({
|
||||
assigneeUserId: null, assigneeAgentId: f.agent.id,
|
||||
});
|
||||
expect((await db.select().from(companyMemberships).where(eq(companyMemberships.id, member.id)))[0].status).toBe("archived");
|
||||
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0);
|
||||
} finally { for (const unsubscribe of unsubscribes) unsubscribe(); }
|
||||
await quiet();
|
||||
});
|
||||
it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => {
|
||||
const f = await fixture(); const dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await expect(db.transaction(async tx => {
|
||||
await notifyNativeStatusWakeWork(tx, { companyId: f.companyId });
|
||||
throw new Error("outer rollback");
|
||||
})).rejects.toThrow("outer rollback");
|
||||
await db.transaction(async tx => {
|
||||
await expect(tx.transaction(async nested => {
|
||||
await notifyNativeStatusWakeWork(nested, { companyId: f.companyId }); throw new Error("nested rollback");
|
||||
})).rejects.toThrow("nested rollback");
|
||||
});
|
||||
await quiet(); expect(dispatchNative).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -19,11 +19,20 @@ import { secretService } from "../services/secrets.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { aiConnectionService } from "../services/ai-connections.js";
|
||||
import * as aiRuntime from "../services/ai-connection-runtime.js";
|
||||
import * as connectorRuntime from "../services/connector-runtime.js";
|
||||
import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js";
|
||||
import * as sentry from "../sentry.js";
|
||||
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
|
||||
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>(complete => { resolve = complete; });
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
const skillFailures = vi.hoisted(() => new Map<string, string>());
|
||||
const skillWaits = vi.hoisted(() => new Map<string, () => Promise<void>>());
|
||||
|
||||
describe("Runner home mode during recovery preparation", () => {
|
||||
it.each([
|
||||
@@ -52,6 +61,7 @@ vi.mock("../services/company-skills.js", async importOriginal => {
|
||||
return { ...service, withRuntimeSkillEntries: async <T>(companyId: string,
|
||||
options: Parameters<typeof service.withRuntimeSkillEntries>[1],
|
||||
consume: (entries: Awaited<ReturnType<typeof service.listRuntimeSkillEntries>>) => T | Promise<T>) => {
|
||||
await skillWaits.get(companyId)?.();
|
||||
const failure = skillFailures.get(companyId);
|
||||
if (failure) throw new Error(failure);
|
||||
return service.withRuntimeSkillEntries(companyId, options, consume);
|
||||
@@ -91,7 +101,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }),
|
||||
});
|
||||
}, 60_000);
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); });
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); skillWaits.clear(); });
|
||||
afterAll(async () => {
|
||||
await heartbeat?.drainActiveRunExecutions();
|
||||
unregisterServerAdapter(adapterType);
|
||||
@@ -288,6 +298,45 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } });
|
||||
}, 60_000);
|
||||
it("overlaps skills with assignments and dependent connection instructions before composition", async () => {
|
||||
const f = await seed();
|
||||
const skillGate = deferred<void>();
|
||||
skillWaits.set(f.companyId, () => skillGate.promise);
|
||||
const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments");
|
||||
const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun");
|
||||
const configured = await configuration(f, { provider: "codex", model: "gpt-6.1-sol", homeMode: "isolated" });
|
||||
const opening = configured.prepare();
|
||||
try {
|
||||
await vi.waitFor(() => {
|
||||
expect(assignments).toHaveBeenCalledOnce();
|
||||
expect(instructions).toHaveBeenCalledOnce();
|
||||
});
|
||||
} finally {
|
||||
skillGate.resolve();
|
||||
}
|
||||
expect(await opening).toMatchObject({ prepared: true, runtimeConfig: { homeMode: "isolated", model: "gpt-6.1-sol" } });
|
||||
});
|
||||
|
||||
it("settles concurrent configuration reads before exposing the first preparation failure", async () => {
|
||||
const f = await seed();
|
||||
const assignmentGate = deferred<Awaited<ReturnType<typeof connectorRuntime.resolveConnectorAssignments>>>();
|
||||
const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments").mockReturnValue(assignmentGate.promise);
|
||||
const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun").mockRejectedValue(new Error("later instruction failure"));
|
||||
skillFailures.set(f.companyId, "first skill failure");
|
||||
const configured = await configuration(f, { provider: "codex", homeMode: "isolated" });
|
||||
let settled = false;
|
||||
const opening = configured.prepare();
|
||||
void opening.then(() => { settled = true; }, () => { settled = true; });
|
||||
try {
|
||||
await vi.waitFor(() => expect(assignments).toHaveBeenCalledOnce());
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
expect(instructions).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
assignmentGate.resolve([]);
|
||||
}
|
||||
await expect(opening).rejects.toThrow("first skill failure");
|
||||
});
|
||||
|
||||
it("does not install new environment declarations when recovering a saved skill selection", async () => {
|
||||
const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`;
|
||||
@@ -445,11 +494,11 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
const runIdHash = createHash("sha256").update(finished.id).digest("hex").slice(0, 12);
|
||||
const steps = exporter.getFinishedSpans().filter(span => span.name.startsWith("heartbeat.prepare.configuration_")
|
||||
&& span.attributes["paperclip.task.run.run_id"] === runIdHash);
|
||||
expect(steps.map(span => span.name)).toEqual([
|
||||
expect(steps.map(span => span.name).sort()).toEqual([
|
||||
"configuration_mentions", "configuration_github_identity", "configuration_secrets", "configuration_ai_runtime",
|
||||
"configuration_skills", "configuration_connector_assignments", "configuration_connector_skills",
|
||||
"configuration_connector_delivery", "configuration_connection_instructions", "configuration_agent_identity",
|
||||
].map(step => `heartbeat.prepare.${step}`));
|
||||
].map(step => `heartbeat.prepare.${step}`).sort());
|
||||
for (const span of steps) {
|
||||
expect(span.status).toEqual({ code: 0 });
|
||||
expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
|
||||
@@ -494,7 +543,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
&& span.attributes["paperclip.task.run.run_id"] === runIdHash);
|
||||
expect(steps.find(span => span.name === "heartbeat.prepare.configuration_ai_runtime")?.status.code).toBe(0);
|
||||
expect(steps.find(span => span.name === "heartbeat.prepare.configuration_skills")?.status).toEqual({ code: 2 });
|
||||
expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(false);
|
||||
expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(true);
|
||||
for (const span of steps) {
|
||||
expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
|
||||
expect(span.events).toEqual([]);
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
|
||||
import { registerRunDispatchWork } from "../services/heartbeat/dispatch-work.js";
|
||||
|
||||
const cleanup: Array<() => Promise<void>> = [];
|
||||
afterEach(async () => {
|
||||
for (const stop of cleanup.splice(0)) await stop();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean } = {}) {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(1_000_000);
|
||||
const dialect = new PgDialect();
|
||||
const execute = vi.fn(async (query: any) => {
|
||||
if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
|
||||
throw new Error("unexpected database query");
|
||||
});
|
||||
const db: { execute: typeof execute; transaction: (callback: (tx: any) => Promise<any>) => Promise<any> } = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise<any>): Promise<any> {
|
||||
return callback({ execute, transaction: db.transaction });
|
||||
} });
|
||||
const state = { due: input.due ?? null as number | null, suppressed: input.suppressed ?? false,
|
||||
draining: input.draining ?? false, standby: false, fail: false };
|
||||
const promote = vi.fn(async () => {
|
||||
if (state.fail) throw new Error("database unavailable");
|
||||
if (state.due !== null && state.due <= Date.now()) state.due = null;
|
||||
});
|
||||
const next = vi.fn(async () => state.due);
|
||||
const dispatch = vi.fn(async (_scope?: { companyId: string }) => {});
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db,
|
||||
canRun: () => !state.draining && !state.standby, canReconcile: () => !state.standby, onError: vi.fn() });
|
||||
const worker = registerRunDispatchWork(coordinator, {
|
||||
promoteDueScheduledRetries: promote, nextScheduledRetryAt: next,
|
||||
resumeQueuedRunsAfterDrain: dispatch, schedulingSuppressed: async () => state.suppressed,
|
||||
}, () => !state.draining && !state.standby);
|
||||
cleanup.push(() => coordinator.stop());
|
||||
return { db, state, promote, next, dispatch, coordinator, worker,
|
||||
notify: (scope?: { companyId: string }) => db.transaction(tx => notifyRunDispatch(tx, scope)),
|
||||
settle: () => vi.advanceTimersByTimeAsync(1_000) };
|
||||
}
|
||||
|
||||
describe("run dispatch obligations", () => {
|
||||
it.each([false, true])("settles empty/feature-suppressed state and stops all owner work (suppressed=%s)", async suppressed => {
|
||||
const f = fixture({ suppressed }); await f.worker.ready;
|
||||
const calls = [f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length];
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect([f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]).toEqual(calls);
|
||||
expect(f.db.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
it("restores an exact deadline and does no earlier recovery work", async () => {
|
||||
const f = fixture({ due: 1_010_000 }); await f.worker.ready;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_010_000);
|
||||
await vi.advanceTimersByTimeAsync(9_999); expect(f.promote).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
expect(f.state.due).toBeNull(); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
expect(f.dispatch).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("rearms earlier work and cancels a future deadline on a committed producer change", async () => {
|
||||
const f = fixture({ due: 1_600_000 }); await f.worker.ready;
|
||||
f.state.due = 1_005_000; await f.notify(); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_005_000);
|
||||
f.state.due = null; await f.notify(); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const before = f.next.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.next).toHaveBeenCalledTimes(before);
|
||||
});
|
||||
it("batches repeated tenant writes and passes only their company to queued dispatch", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
for (let index = 0; index < 100; index++) {
|
||||
const source = { companyId: "company-a", issueId: `issue-${index % 2}`,
|
||||
runId: `run-${index}`, agentId: `agent-${index}`, interactionBody: "not a dispatch key" };
|
||||
await f.notify(source);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
}
|
||||
expect(f.promote.mock.calls.length).toBeLessThanOrEqual(2);
|
||||
expect(f.dispatch.mock.calls.length).toBeLessThanOrEqual(2);
|
||||
await f.settle();
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ companyId: "company-a" });
|
||||
for (const [scope] of f.dispatch.mock.calls.slice(1)) expect(scope).toEqual({ companyId: "company-a" });
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const count = f.dispatch.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.dispatch).toHaveBeenCalledTimes(count);
|
||||
});
|
||||
it("remembers the first producer arriving during drain without owner SQL", async () => {
|
||||
const f = fixture(); await f.worker.ready; f.state.draining = true;
|
||||
await f.notify(); await f.settle(); await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).toHaveBeenCalledOnce();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(2); expect(f.dispatch).toHaveBeenCalledTimes(2);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("releases a failed producer hold during drain and retains recovery for resume", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
f.state.fail = true; await f.notify(); await f.settle();
|
||||
expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
f.state.draining = true; await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(2); expect(f.next).toHaveBeenCalledOnce();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.fail = false; f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("preserves a notification during a nonoverlapping pass", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>(resolve => { release = resolve; });
|
||||
f.promote.mockImplementationOnce(() => held);
|
||||
await f.notify(); await f.settle();
|
||||
await f.notify(); await f.settle();
|
||||
expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
release(); await f.settle();
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("does not dispatch on warm standby and cancels deadlines at shutdown", async () => {
|
||||
const f = fixture({ due: 1_010_000 }); f.state.standby = true; await f.worker.ready;
|
||||
await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).not.toHaveBeenCalled();
|
||||
f.state.standby = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledOnce(); expect(f.state.due).toBeNull();
|
||||
f.state.due = Date.now() + 60_000; await f.notify(); await f.settle(); await f.worker.stop();
|
||||
await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,238 @@
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, costEvents, createDb, databaseWorkPending, heartbeatRuns } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { agentService } from "../services/agents.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_DISPATCH_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("run dispatch production wiring (controlled local process adapter)", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let worker: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]> | undefined;
|
||||
let reads: ReturnType<typeof vi.spyOn>;
|
||||
let draining = false;
|
||||
const errors: unknown[] = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("run-dispatch-work-");
|
||||
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`TRUNCATE companies CASCADE`);
|
||||
heartbeat = heartbeatService(db); draining = false; errors.length = 0;
|
||||
reads = vi.spyOn(db, "select");
|
||||
});
|
||||
afterEach(async () => {
|
||||
await worker?.stop(); await coordinator?.stop();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
worker = undefined; coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining,
|
||||
canReconcile: () => true, onError: error => errors.push(error) });
|
||||
worker = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
|
||||
await worker.ready;
|
||||
}
|
||||
async function fixture() {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Dispatch", issuePrefix: `D${companyId.slice(0, 7)}`,
|
||||
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "dispatch-user" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Controlled process", role: "engineer", status: "idle",
|
||||
adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('dispatch-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
|
||||
}).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "failed",
|
||||
responsibleUserId: "dispatch-user", errorCode: "codex_transient_upstream", finishedAt: new Date(),
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, contextSnapshot: {},
|
||||
}).returning();
|
||||
return { companyId, agentId: agent.id, runId: run.id };
|
||||
}
|
||||
async function quiet() {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull(), { timeout: 5_000 });
|
||||
const count = reads.mock.calls.length;
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
} finally { vi.useRealTimers(); }
|
||||
}
|
||||
async function status(id: string) {
|
||||
return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]);
|
||||
}
|
||||
async function completed(id: string) {
|
||||
await vi.waitFor(async () => expect(["running", "succeeded"]).toContain((await status(id))?.status), { timeout: 10_000 });
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const completed = await status(id);
|
||||
expect(completed.status).toBe("succeeded");
|
||||
expect(completed.stdoutExcerpt).toContain("dispatch-proof");
|
||||
}
|
||||
it("keeps fresh empty, configured idle and retained terminal history quiet", async () => {
|
||||
await start(); await quiet();
|
||||
await fixture(); await worker!.stop(); await coordinator!.stop();
|
||||
await start(); await quiet();
|
||||
});
|
||||
it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 4_000, random: () => 0 });
|
||||
expect(result.outcome).toBe("scheduled");
|
||||
if (result.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime()), { timeout: 5_000 });
|
||||
expect((await status(result.run.id)).status).toBe("scheduled_retry");
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(Date.now()).toBeLessThan(result.dueAt.getTime());
|
||||
expect((await status(result.run.id)).status).toBe("scheduled_retry");
|
||||
await completed(result.run.id);
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(1);
|
||||
await quiet();
|
||||
});
|
||||
it("reconstructs an overdue retry after process restart without a periodic tick", async () => {
|
||||
const f = await fixture();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 120_000), delayMs: 60_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
heartbeat = heartbeatService(db);
|
||||
await start(); await completed(scheduled.run.id); await quiet();
|
||||
});
|
||||
it("cancels a future retry and removes its old deadline", async () => {
|
||||
const f = await fixture(); await start();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime()), { timeout: 5_000 });
|
||||
await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true });
|
||||
await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled");
|
||||
});
|
||||
it("retains the first retry written during drain and catches it up after resume", async () => {
|
||||
const f = await fixture(); await start(); await quiet(); draining = true;
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 200 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
const count = reads.mock.calls.length;
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
} finally { vi.useRealTimers(); }
|
||||
expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
|
||||
draining = false; worker!.wake();
|
||||
await completed(scheduled.run.id); await quiet();
|
||||
});
|
||||
it("uses a committed budget raise to resume a queued run in the company it paused", async () => {
|
||||
const f = await fixture();
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 100,
|
||||
metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
|
||||
const [cost] = await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
provider: "fixture", biller: "fixture", model: "controlled", inputTokens: 0, outputTokens: 0,
|
||||
costCents: 125, occurredAt: new Date() }).returning();
|
||||
await budgets.evaluateCostEvent(cost);
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0].status).toBe("paused");
|
||||
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
|
||||
await start(); await quiet(); expect((await status(queued.id)).status).toBe("queued");
|
||||
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 10_000,
|
||||
metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
|
||||
await completed(queued.id); await quiet();
|
||||
});
|
||||
it("clearing an agent error wakes saved queue work only after its own transaction commits", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
await db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId));
|
||||
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
|
||||
let writing: ReturnType<ReturnType<typeof agentService>["clearError"]> | undefined;
|
||||
try {
|
||||
await db.transaction(async blocker => {
|
||||
await blocker.select().from(agents).where(eq(agents.id, f.agentId)).for("update");
|
||||
writing = agentService(db).clearError(f.agentId);
|
||||
void writing.catch(() => {});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(false);
|
||||
expect((await status(queued.id)).status).toBe("queued");
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
|
||||
});
|
||||
} finally { await writing; }
|
||||
await completed(queued.id); await quiet();
|
||||
});
|
||||
it("leaves cancellation-owned retries quiet even when their old promotion time is overdue", async () => {
|
||||
const f = await fixture();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 60_000), delayMs: 1_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await db.update(heartbeatRuns).set({ resultJson: { startupCancellation: { requestedAt: new Date().toISOString() },
|
||||
cancellation: { expected: true, reason: "operator stop", source: "operator" } } }).where(eq(heartbeatRuns.id, scheduled.run.id));
|
||||
await start(); await quiet();
|
||||
expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
|
||||
expect(await heartbeat.promoteDueScheduledRetries()).toEqual({ promoted: 0, runIds: [] });
|
||||
await quiet();
|
||||
});
|
||||
it("rejects an error clear without publishing work when the lifecycle is no longer ready", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
await db.update(agents).set({ status: "error", lifecycleState: "paused" }).where(eq(agents.id, f.agentId));
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await expect(agentService(db).clearError(f.agentId)).rejects.toThrow("Only agents in error status");
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
const count = reads.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
} finally { vi.useRealTimers(); }
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
|
||||
await quiet();
|
||||
});
|
||||
it("keeps a foreign company's future retry intact while concurrent workers claim one due retry", async () => {
|
||||
const selected = await fixture(), foreign = await fixture(); await start();
|
||||
const later = await heartbeat.scheduleBoundedRetry(foreign.runId, { delayMs: 600_000 });
|
||||
const due = await heartbeat.scheduleBoundedRetry(selected.runId, { delayMs: 500 });
|
||||
if (later.outcome !== "scheduled" || due.outcome !== "scheduled") throw new Error("Expected retries");
|
||||
const other = heartbeatService(db);
|
||||
const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const competingWorker = other.registerRunDispatchWork(competing, () => true);
|
||||
try {
|
||||
await competingWorker.ready; await completed(due.run.id); await other.drainActiveRunExecutions();
|
||||
expect((await status(later.run.id)).status).toBe("scheduled_retry");
|
||||
expect((await status(later.run.id)).scheduledRetryAt).toEqual(later.dueAt);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, selected.runId))).toHaveLength(1);
|
||||
await heartbeat.cancelRun(later.run.id, "fixture teardown", { suppressImmediateRecovery: true });
|
||||
await quiet();
|
||||
} finally { await competingWorker.stop(); await competing.stop(); await other.drainActiveRunExecutions(); }
|
||||
});
|
||||
it("holds nested producer work until outer commit and rolls back without a successor", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
|
||||
let queuedId = "";
|
||||
const writing = db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await notifyRunDispatch(nested);
|
||||
const [row] = await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
responsibleUserId: "dispatch-user", status: "queued", contextSnapshot: {} }).returning();
|
||||
queuedId = row.id;
|
||||
});
|
||||
await held;
|
||||
});
|
||||
await vi.waitFor(() => expect(queuedId).not.toBe(""));
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(await status(queuedId)).toBeUndefined();
|
||||
release(); await writing; await completed(queuedId); await quiet();
|
||||
await expect(db.transaction(async tx => {
|
||||
await tx.transaction(async nested => { await notifyRunDispatch(nested); await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "queued" }); });
|
||||
throw new Error("rollback outer");
|
||||
})).rejects.toThrow("rollback outer");
|
||||
await quiet();
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,11 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared";
|
||||
import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { prepareHeartbeatTaskContext } from "../services/heartbeat/task-context.js";
|
||||
import { documentService } from "../services/documents.js";
|
||||
import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js";
|
||||
import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js";
|
||||
@@ -19,6 +20,7 @@ if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${s
|
||||
describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
const adapterType = "task_context_test";
|
||||
const captured = new Map<string, AdapterExecutionContext>();
|
||||
const inspectPreparation = new Map<string, () => void>();
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
@@ -33,6 +35,7 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
type: adapterType,
|
||||
supportsLocalAgentJwt: false,
|
||||
execute: async (input) => {
|
||||
inspectPreparation.get(input.agent.id)?.();
|
||||
captured.set(input.runId, input);
|
||||
return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} };
|
||||
},
|
||||
@@ -118,6 +121,101 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reads selected comments once for both wake projections", async () => {
|
||||
const f = await seed();
|
||||
const commentId = randomUUID();
|
||||
await db.insert(issueComments).values({
|
||||
id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user",
|
||||
body: "selected wake canary",
|
||||
});
|
||||
const select = vi.spyOn(db, "select");
|
||||
inspectPreparation.set(f.agentId, () => {
|
||||
const commentReads = select.mock.calls.filter(([projection]) => projection
|
||||
&& projection.body === issueComments.body && projection.id === issueComments.id && projection.deletedAt === issueComments.deletedAt);
|
||||
expect(commentReads).toHaveLength(1);
|
||||
});
|
||||
try {
|
||||
const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] });
|
||||
expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "selected wake canary" });
|
||||
expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "selected wake canary" }] });
|
||||
} finally {
|
||||
inspectPreparation.delete(f.agentId);
|
||||
select.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels deleted-only queued comments before adapter dispatch", async () => {
|
||||
const f = await seed(), commentId = randomUUID();
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "discarded wake", deletedAt: new Date() });
|
||||
const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented",
|
||||
payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] } });
|
||||
expect(run).not.toBeNull();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const finished = await heartbeat.getRun(run!.id);
|
||||
expect(finished).toMatchObject({ status: "cancelled", error: "Queued messages were discarded before dispatch" });
|
||||
expect(captured.has(run!.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves deleted comment metadata without delivering its body", async () => {
|
||||
const f = await seed();
|
||||
const commentId = randomUUID();
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "deleted wake canary", deletedAt: new Date(),
|
||||
presentation: { fixture: "deleted" }, metadata: { fixture: "deleted" } });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
|
||||
await prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
|
||||
policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
|
||||
});
|
||||
expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "", presentation: null, metadata: null });
|
||||
expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "", presentation: null, metadata: null }] });
|
||||
expect(context.paperclipTaskMarkdown).not.toContain("deleted wake canary");
|
||||
});
|
||||
|
||||
it.each([false, true])("uses admitted skill-test scope when recovery is %s", async (recovering) => {
|
||||
const f = await seed();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } };
|
||||
const pinned = vi.fn(async () => null);
|
||||
const preparing = prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
|
||||
policy: { recovering, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: pinned },
|
||||
});
|
||||
if (recovering) await expect(preparing).rejects.toThrow("skill test's saved version is unavailable");
|
||||
else await preparing;
|
||||
expect(pinned).toHaveBeenCalledTimes(recovering ? 1 : 0);
|
||||
expect(context.paperclipAgentJwtScope).toEqual({ kind: recovering ? "skill_test" : "standard", ...(recovering ? { issueId: f.issueId } : {}) });
|
||||
});
|
||||
|
||||
it("retains the context issue scope when no issue reference is available", async () => {
|
||||
const f = await seed();
|
||||
const otherIssueId = randomUUID(), commentId = randomUUID();
|
||||
await db.insert(issues).values({ id: otherIssueId, companyId: f.companyId, title: "Other issue", issueNumber: 2 });
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: otherIssueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "other issue canary" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
|
||||
await prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef: null, issueContext: null, context, taskSession: null },
|
||||
policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
|
||||
});
|
||||
expect(JSON.stringify(context.paperclipWake)).not.toContain("other issue canary");
|
||||
expect(context.paperclipWakeComment).toBeUndefined();
|
||||
});
|
||||
|
||||
it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => {
|
||||
const f = await seed();
|
||||
const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId });
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db";
|
||||
import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
|
||||
@@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
|
||||
(support.supported ? describe : describe.skip)("historical cleanup verification", () => {
|
||||
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true);
|
||||
const state = async () => (await read()).backgroundWork;
|
||||
@@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
|
||||
inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }),
|
||||
});
|
||||
}
|
||||
it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => {
|
||||
const f = await seed(adHoc);
|
||||
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
|
||||
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
|
||||
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value);
|
||||
let absent = false;
|
||||
const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => {
|
||||
if (!absent) return Response.json({ absent: false });
|
||||
const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000);
|
||||
expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id,
|
||||
leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId });
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url");
|
||||
const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1",
|
||||
sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url");
|
||||
return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` });
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchImpl);
|
||||
const runtime = environmentRuntimeService(db);
|
||||
const service = historicalCleanupVerificationService(db, {
|
||||
inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined,
|
||||
});
|
||||
expect(await state()).toBe("present");
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 });
|
||||
expect(await state()).toBe("present");
|
||||
absent = true;
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
|
||||
expect(await state()).toBe("none");
|
||||
const [current] = await db.select().from(environmentLeases);
|
||||
expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId });
|
||||
expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination");
|
||||
expect(await service.sweep()).toMatchObject({ checked: 0 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => {
|
||||
await seed(true);
|
||||
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
|
||||
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value);
|
||||
const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl);
|
||||
const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
|
||||
acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null,
|
||||
inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }),
|
||||
}] });
|
||||
const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined });
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 });
|
||||
expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => {
|
||||
const f = await seed(adHoc); expect(await state()).toBe("present");
|
||||
const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
|
||||
import { registerIssueRecoveryWork } from "../services/recovery/work.js";
|
||||
import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
|
||||
import { taskWatchdogService } from "../services/task-watchdogs.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("issue recovery production producers and deadlines", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
let worker: ReturnType<ReturnType<typeof heartbeatService>["registerIssueStateRecoveryWork"]>;
|
||||
let dispatch: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]>;
|
||||
let cancellation: ReturnType<ReturnType<typeof heartbeatService>["registerScheduledRetryCancellationWork"]>;
|
||||
let draining = false;
|
||||
const tasks = new Map<string, { nextRunAt?: () => number | null; hasPending: () => Promise<boolean> }>();
|
||||
const errors: unknown[] = [];
|
||||
const passes = new Set<Promise<unknown>>();
|
||||
const extraStops: Array<() => Promise<void>> = [];
|
||||
const extraDeadlines: Array<() => number | null> = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-");
|
||||
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`TRUNCATE companies CASCADE`);
|
||||
heartbeat = heartbeatService(db); draining = false; errors.length = 0;
|
||||
});
|
||||
afterEach(async () => {
|
||||
for (const stop of extraStops.splice(0)) await stop();
|
||||
extraDeadlines.length = 0;
|
||||
await worker?.stop(); await dispatch?.stop(); await cancellation?.stop(); await coordinator?.stop();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
tasks.clear();
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => {
|
||||
tasks.set(queue, task);
|
||||
return register(queue, { ...task, run: signal => {
|
||||
const work = task.run(signal); passes.add(work);
|
||||
void work.then(() => passes.delete(work), () => passes.delete(work));
|
||||
return work;
|
||||
} });
|
||||
};
|
||||
worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining);
|
||||
dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
|
||||
cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining);
|
||||
await Promise.all([worker.ready, dispatch.ready, cancellation.ready]);
|
||||
}
|
||||
async function compete() {
|
||||
const other = heartbeatService(db);
|
||||
const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const work = other.registerIssueStateRecoveryWork(competing, () => true);
|
||||
extraStops.push(() => competing.stop()); extraDeadlines.push(competing.nextWakeAt); await work.ready;
|
||||
}
|
||||
async function fixture() {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: "recovery-user", name: "Recovery operator", email: "recovery-user@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 7)}`,
|
||||
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "recovery-user" });
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: "recovery-user", principalType: "user", status: "active", membershipRole: "owner" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Recovery process", role: "engineer", status: "idle",
|
||||
adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('recovery-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
|
||||
}).returning();
|
||||
return { companyId, agentId: agent.id };
|
||||
}
|
||||
async function settled() {
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const wake = coordinator.nextWakeAt();
|
||||
const detail = wake === null ? "" : JSON.stringify({ now: Date.now(), tasks: await Promise.all([...tasks].map(async ([queue, task]) => ({ queue, at: task.nextRunAt?.(), pending: await task.hasPending() }))),
|
||||
runs: (await db.select().from(heartbeatRuns)).map(run => ({ id: run.id, status: run.status, error: run.errorCode, lease: run.controllerLeaseExpiresAt, reason: run.contextSnapshot?.wakeReason })) });
|
||||
expect(wake, detail).toBeNull();
|
||||
expect(extraDeadlines.map(next => next())).toEqual(extraDeadlines.map(() => null));
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
}, { timeout: 40_000 });
|
||||
}
|
||||
async function quiet() {
|
||||
await settled();
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
|
||||
const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute");
|
||||
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); }
|
||||
finally { vi.useRealTimers(); reads.mockRestore(); executes.mockRestore(); }
|
||||
}
|
||||
async function finish(issueId: string) {
|
||||
await issueService(db).update(issueId, { status: "done" });
|
||||
// Closing one blocker can legitimately start its dependent. Wait for this
|
||||
// task's dispatch and execution; quiet() verifies the whole fixture later.
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(and(
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
|
||||
sql`${heartbeatRuns.status} in ('queued', 'running')`,
|
||||
))).toHaveLength(0);
|
||||
}, { timeout: 15_000 });
|
||||
const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
if (saved.length > 0) {
|
||||
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
|
||||
// Keep PostgreSQL I/O and coordinator settlement ahead of virtual time.
|
||||
// A single leap can otherwise occur before the retry deadline is armed.
|
||||
while (Date.now() <= due) {
|
||||
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
|
||||
await Promise.allSettled([...passes]);
|
||||
await immediate();
|
||||
}
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
|
||||
expect(remaining).toHaveLength(0);
|
||||
});
|
||||
for (const retry of saved) {
|
||||
const [settledRetry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retry.id));
|
||||
expect(settledRetry.status).toBe("cancelled");
|
||||
}
|
||||
}
|
||||
}
|
||||
async function completed(issueId: string) {
|
||||
let runId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const runs = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`);
|
||||
const run = runs.find(row => row.status === "running" || row.status === "succeeded");
|
||||
expect(run).toBeDefined(); runId = run!.id;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(current.status, JSON.stringify({ error: current.error, code: current.errorCode, stderr: current.stderrExcerpt })).toBe("succeeded");
|
||||
}, { timeout: 15_000 });
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.stdoutExcerpt).toContain("recovery-proof");
|
||||
return run;
|
||||
}
|
||||
it("keeps empty, configured idle, and terminal history quiet", async () => {
|
||||
await fixture(); await start(); await quiet();
|
||||
});
|
||||
it("creates a blocking child through the ordinary root service transaction", async () => {
|
||||
const f = await fixture();
|
||||
const parent = await issueService(db).create(f.companyId, { title: "Parent", status: "todo" });
|
||||
const child = await issueService(db).createChild(parent.id, { title: "Blocking child", status: "todo", blockParentUntilDone: true });
|
||||
expect(child.parentBlockerAdded).toBe(true);
|
||||
expect(await db.select().from(issueRelations).where(and(eq(issueRelations.companyId, f.companyId),
|
||||
eq(issueRelations.issueId, child.issue.id), eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1);
|
||||
});
|
||||
it.each(["startup", "committed"])("restores blocker-only and completed-action debt with no assignment candidates (%s)", async mode => {
|
||||
const f = await fixture();
|
||||
if (mode === "committed") { await start(); await quiet(); }
|
||||
const blocker = await issueService(db).create(f.companyId, { title: "Orphan blocker", status: "blocked", createdByAgentId: f.agentId });
|
||||
const dependent = await issueService(db).create(f.companyId, { title: "Blocked dependent", status: "blocked", blockedByIssueIds: [blocker.id] });
|
||||
const completedSource = await issueService(db).create(f.companyId, { title: "Completed source", status: "done" });
|
||||
const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: completedSource.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
|
||||
fingerprint: "source-terminal", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
|
||||
if (mode === "startup") await start();
|
||||
await vi.waitFor(async () => {
|
||||
expect((await db.select().from(issues).where(eq(issues.id, blocker.id)))[0].assigneeAgentId).toBe(f.agentId);
|
||||
expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
|
||||
}, { timeout: 40_000 });
|
||||
await finish(blocker.id); await finish(dependent.id); await quiet();
|
||||
});
|
||||
it("bounds subsidiary scans for100 assignment candidates and coalesces repeated company writes", async () => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
await db.update(agents).set({ lifecycleState: "preparing" }).where(eq(agents.id, f.agentId));
|
||||
await db.insert(issues).values(Array.from({ length: 100 }, (_, index) => ({ companyId: f.companyId,
|
||||
title: `Waiting setup ${index}`, status: "todo", assigneeAgentId: f.agentId })));
|
||||
const [source] = await db.insert(issues).values({ companyId: f.companyId, title: "Done source", status: "done" }).returning();
|
||||
const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: source.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
|
||||
fingerprint: "query-budget", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
|
||||
const scans = { blockers: 0, actions: 0, loads: 0 };
|
||||
const originalSelect = db.select.bind(db);
|
||||
vi.spyOn(db, "select").mockImplementation(((fields?: Record<string, unknown>) => {
|
||||
const query = originalSelect(fields as never); const from = query.from.bind(query);
|
||||
query.from = ((table: unknown) => {
|
||||
if (table === issues && fields?.id === issues.id && fields.companyId === issues.companyId && Object.keys(fields).length === 2) scans.loads++;
|
||||
if (table === issueRelations && fields?.createdByAgentId === issues.createdByAgentId) scans.blockers++;
|
||||
if (table === issueRecoveryActions && fields?.action === issueRecoveryActions) scans.actions++;
|
||||
return from(table as never);
|
||||
}) as typeof query.from;
|
||||
return query;
|
||||
}) as typeof db.select);
|
||||
await start(); await settled();
|
||||
expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(8);
|
||||
expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
|
||||
const [foreignSource] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign done", status: "done" }).returning();
|
||||
const [foreignAction] = await db.insert(issueRecoveryActions).values({ companyId: foreign.companyId, sourceIssueId: foreignSource.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "foreign-query-budget", nextAction: "Inspect" }).returning();
|
||||
const before = { ...scans };
|
||||
for (let index = 0; index < 100; index++) await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(scans.loads).toBe(before.loads);
|
||||
await settled();
|
||||
expect(scans.actions).toBe(before.actions); expect(scans.blockers).toBe(before.blockers);
|
||||
expect(scans.loads).toBeLessThanOrEqual(before.loads + 4);
|
||||
expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active");
|
||||
expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet();
|
||||
});
|
||||
it("limits an ordinary comment to its issue among one thousand open assignments", async () => {
|
||||
const f = await fixture();
|
||||
const rows = await db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({
|
||||
companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId,
|
||||
}))).returning({ id: issues.id });
|
||||
const checked: string[] = [];
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const pass = task.run(signal); passes.add(pass);
|
||||
void pass.then(() => passes.delete(pass), () => passes.delete(pass));
|
||||
return pass;
|
||||
} });
|
||||
// Observe the actual paged reconstruction boundary without executing one
|
||||
// thousand adapters. The producer, transaction, coordinator and SQL are real;
|
||||
// the other acceptance cases exercise the resulting recovery decisions.
|
||||
worker = registerIssueRecoveryWork(db, coordinator, {
|
||||
stranded: async scope => { checked.push(scope.issueId); },
|
||||
orphanBlockers: async () => {}, recoveryActions: async () => {},
|
||||
dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {},
|
||||
watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false,
|
||||
}, () => true);
|
||||
await worker.ready;
|
||||
async function settleWork() {
|
||||
for (let step = 0; step < 120; step++) {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await Promise.allSettled([...passes]); await immediate();
|
||||
if (coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return;
|
||||
}
|
||||
throw new Error("Recovery did not settle its committed work within two virtual minutes");
|
||||
}
|
||||
await settleWork();
|
||||
expect(new Set(checked).size).toBe(1000);
|
||||
checked.length = 0;
|
||||
await issueService(db).addComment(rows[0].id, "New evidence on this task", { userId: "recovery-user" });
|
||||
await settleWork();
|
||||
expect(checked).toEqual([rows[0].id]);
|
||||
// Company eligibility changes still reconstruct all assignments. Narrowing
|
||||
// comment hints must not hide unrelated work after an operator resumes it.
|
||||
checked.length = 0;
|
||||
await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
|
||||
await settleWork();
|
||||
expect(new Set(checked).size).toBe(1000);
|
||||
await quiet();
|
||||
});
|
||||
it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId });
|
||||
await completed(issue.id);
|
||||
await finish(issue.id); await quiet();
|
||||
}, 90_000);
|
||||
it("reacts to blocker resolution once and dispatches the durable dependency wake", async () => {
|
||||
const f = await fixture();
|
||||
const blocker = await issueService(db).create(f.companyId, { title: "Blocker", status: "todo" });
|
||||
const dependent = await issueService(db).create(f.companyId, { title: "Dependent", status: "blocked", assigneeAgentId: f.agentId,
|
||||
blockedByIssueIds: [blocker.id] });
|
||||
await start(); await quiet();
|
||||
await compete();
|
||||
await issueService(db).update(blocker.id, { status: "done" });
|
||||
const run = await completed(dependent.id);
|
||||
expect(run.contextSnapshot?.wakeReason).toBe("issue_blockers_resolved");
|
||||
await finish(dependent.id); await quiet();
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_blockers_resolved'`))).toHaveLength(1);
|
||||
}, 90_000);
|
||||
it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Finished subtree", status: "done" });
|
||||
await start(); await quiet();
|
||||
await compete();
|
||||
const watchdog = await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
|
||||
let reviewId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const [row] = await db.select().from(issueWatchdogs).where(eq(issueWatchdogs.id, watchdog.watchdog.id));
|
||||
expect(row.watchdogIssueId).toBeTruthy(); reviewId = row.watchdogIssueId!;
|
||||
}, { timeout: 15_000 });
|
||||
await completed(reviewId!);
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1);
|
||||
await finish(reviewId!);
|
||||
await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
}, 90_000);
|
||||
it("bounds ordinary writes to their company and reconstructs foreign stale debt after restart", async () => {
|
||||
const selected = await fixture(); await start(); await quiet();
|
||||
const foreign = await fixture();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: foreign.companyId, agentId: foreign.agentId,
|
||||
status: "failed", finishedAt: new Date() }).returning();
|
||||
const [stale] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign stale lock", status: "done",
|
||||
checkoutRunId: run.id }).returning();
|
||||
await issueService(db).create(selected.companyId, { title: "Selected change", status: "done" }); await quiet();
|
||||
expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBe(run.id);
|
||||
await worker.stop(); await dispatch.stop(); await cancellation.stop(); await coordinator.stop();
|
||||
heartbeat = heartbeatService(db); await start(); await quiet();
|
||||
expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBeNull();
|
||||
});
|
||||
it("fences nested assignment writes until outer commit and ignores rollback", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
let committedId = "";
|
||||
await db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Nested commit", status: "todo", assigneeAgentId: f.agentId }, nested);
|
||||
committedId = issue.id;
|
||||
});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
|
||||
});
|
||||
await completed(committedId); await finish(committedId); await settled();
|
||||
await expect(db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
await issueService(db).create(f.companyId, { title: "Nested rollback", status: "todo", assigneeAgentId: f.agentId }, nested);
|
||||
});
|
||||
throw new Error("rollback control");
|
||||
})).rejects.toThrow("rollback control");
|
||||
await quiet();
|
||||
expect(await db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0);
|
||||
}, 90_000);
|
||||
it("retains the first committed assignment during drain and executes on resume", async () => {
|
||||
const f = await fixture(); await start(); await quiet(); draining = true;
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Drained assignment", status: "todo", assigneeAgentId: f.agentId });
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
const reads = vi.spyOn(db, "select");
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try { await vi.advanceTimersByTimeAsync(90_000); expect(reads).not.toHaveBeenCalled(); }
|
||||
finally { vi.useRealTimers(); reads.mockRestore(); }
|
||||
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
|
||||
draining = false; worker.wake(); dispatch.wake();
|
||||
await completed(issue.id); await finish(issue.id); await quiet();
|
||||
}, 90_000);
|
||||
it("retains a fresh subtree's exact first-run grace then disarms on disable", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Fresh unassigned subtree", status: "todo" });
|
||||
await start(); await quiet();
|
||||
await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(source.createdAt.getTime() + 15_000), { timeout: 5_000 });
|
||||
expect(await db.select().from(issues).where(eq(issues.originKind, "task_watchdog"))).toHaveLength(0);
|
||||
await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
});
|
||||
it("waits for a live foreign controller's exact stored lease before clearing a terminal lock", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Remote owner", status: "done" });
|
||||
const lease = new Date(Date.now() + 5_000);
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running",
|
||||
runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: lease,
|
||||
contextSnapshot: { issueId: source.id }, processPid: 2_000_000_000, startedAt: new Date() }).returning();
|
||||
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, source.id));
|
||||
await start();
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.getTime()), { timeout: 3_000 });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running");
|
||||
await vi.waitFor(async () => expect((await db.select().from(issues).where(eq(issues.id, source.id)))[0].executionRunId).toBeNull(), { timeout: 12_000 });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded");
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
}, 90_000);
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { issueRecoveryActions, issueWatchdogs } from "@paperclipai/db";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
|
||||
import { registerIssueRecoveryWork } from "../services/recovery/work.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
const stops: Array<() => Promise<void>> = [];
|
||||
afterEach(async () => { for (const stop of stops.splice(0)) await stop(); vi.useRealTimers(); });
|
||||
function fixture() {
|
||||
vi.useFakeTimers(); vi.setSystemTime(1_000_000);
|
||||
const dialect = new PgDialect();
|
||||
const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }>, blockers: [] as Array<{ id: string; companyId: string }>, actions: [] as Array<{ id: string; companyId: string }> };
|
||||
const execute = vi.fn(async (query: any) => {
|
||||
if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
|
||||
throw new Error("unexpected SQL");
|
||||
});
|
||||
const load = vi.fn((table: unknown, distinct = false) => distinct
|
||||
? table === issueRecoveryActions ? rows.actions : rows.blockers
|
||||
: table === issueWatchdogs ? rows.watchdogs : rows.issues);
|
||||
const query = (distinct = false) => ({ from: (table: unknown) => ({ where: () => ({ orderBy: () => ({ limit: async () => load(table, distinct) }) }) }) });
|
||||
const db: any = installDatabaseWorkSignals({ execute,
|
||||
select: () => query(), selectDistinct: () => query(true),
|
||||
transaction: async (callback: (tx: any) => Promise<unknown>) => callback({ execute, select: db.select, selectDistinct: db.selectDistinct, transaction: db.transaction }),
|
||||
});
|
||||
const state = { draining: false, standby: false, suppressed: false, deadline: null as number | null, fail: false, incomplete: false };
|
||||
const check = vi.fn(async (scope: { onDeferredDeadline(at: number): void }) => {
|
||||
if (state.fail) throw new Error("database unavailable");
|
||||
if (state.deadline !== null && state.deadline > Date.now()) scope.onDeferredDeadline(state.deadline);
|
||||
});
|
||||
const orphanBlockers = vi.fn(async () => {}), recoveryActions = vi.fn(async () => {});
|
||||
const suppression = vi.fn(async () => state.suppressed);
|
||||
const owner = { orphanBlockers, recoveryActions, stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; },
|
||||
locks: check, watchdogs: async (scope: any) => { await check(scope); return { incomplete: state.incomplete ? 1 : 0 }; },
|
||||
schedulingSuppressed: suppression };
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.standby,
|
||||
canReconcile: () => !state.standby, onError: vi.fn() });
|
||||
const worker = registerIssueRecoveryWork(db, coordinator, owner, () => !state.draining && !state.standby);
|
||||
stops.push(() => coordinator.stop());
|
||||
return { db, rows, state, check, load, coordinator, worker, orphanBlockers, recoveryActions, suppression,
|
||||
notify: () => db.transaction((tx: any) => notifyIssueRecovery(tx, { companyId: "selected" })),
|
||||
settle: () => vi.advanceTimersByTimeAsync(1_000) };
|
||||
}
|
||||
describe("issue recovery obligation lifecycle", () => {
|
||||
it("has no timer, SQL, or owner probe after empty reconstruction", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull(); const loaded = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
|
||||
});
|
||||
it("reads scheduling suppression once per pass instead of once per issue", async () => {
|
||||
const f = fixture();
|
||||
f.rows.issues.push(...Array.from({ length: 25 }, (_, index) => ({ id: `issue-${index}`, companyId: "selected" })));
|
||||
await f.worker.ready;
|
||||
expect(f.check).toHaveBeenCalledTimes(75);
|
||||
expect(f.suppression).toHaveBeenCalledTimes(6);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("backs off a failed suppression read without probing an empty or failed owner repeatedly", async () => {
|
||||
const f = fixture(); f.suppression.mockRejectedValueOnce(new Error("settings unavailable"));
|
||||
await f.worker.ready; const reads = f.suppression.mock.calls.length;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
await vi.advanceTimersByTimeAsync(29_999); expect(f.suppression).toHaveBeenCalledTimes(reads);
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.suppression).toHaveBeenCalledTimes(reads + 1);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("normalizes a wide scope with cyclic SQL objects before publishing work hints", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
const wide = { companyId: "selected", endpointId: issueRecoveryActions.id, table: issueRecoveryActions };
|
||||
await f.db.transaction((tx: object) => notifyIssueRecovery(tx, wide)); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const loads = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.load).toHaveBeenCalledTimes(loads);
|
||||
});
|
||||
it("runs orphan blocker and action duties without assignment entries", async () => {
|
||||
const f = fixture(); f.rows.blockers.push({ id: "selected", companyId: "selected" });
|
||||
f.rows.actions.push({ id: "selected", companyId: "selected" }); await f.worker.ready;
|
||||
expect(f.check).not.toHaveBeenCalled();
|
||||
expect(f.orphanBlockers).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
|
||||
expect(f.recoveryActions).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("coalesces repeated company reconstruction without delaying an existing lease deadline", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
f.state.deadline = 1_015_000; await f.worker.ready;
|
||||
await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length, checked = f.check.mock.calls.length;
|
||||
for (let index = 0; index < 100; index++) { await f.notify(); await vi.advanceTimersByTimeAsync(10); }
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).toHaveBeenCalledTimes(checked);
|
||||
await vi.advanceTimersByTimeAsync(13_000);
|
||||
expect(f.state.deadline).toBe(1_015_000);
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_031_000);
|
||||
await vi.advanceTimersByTimeAsync(16_000);
|
||||
expect(f.load.mock.calls.length).toBeGreaterThan(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("retains exact grace and lease deadlines without early owner probes", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.deadline = 1_015_000;
|
||||
await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_015_000);
|
||||
const checked = f.check.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(14_999); expect(f.check).toHaveBeenCalledTimes(checked);
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.check.mock.calls.length).toBe(checked * 2);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("releases settled producer holds during domain drain while coordinator remains open", async () => {
|
||||
const f = fixture(); await f.worker.ready; f.state.draining = true;
|
||||
f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.draining = false; f.worker.wake(); await f.settle();
|
||||
expect(f.check).toHaveBeenCalled(); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("retries concrete failed entries and then disarms without reloading the company", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); f.state.fail = true;
|
||||
await f.worker.ready; const loaded = f.load.mock.calls.length;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
await vi.advanceTimersByTimeAsync(29_999); const checked = f.check.mock.calls.length;
|
||||
f.state.fail = false; await vi.advanceTimersByTimeAsync(1);
|
||||
expect(f.check.mock.calls.length).toBe(checked * 2); expect(f.load).toHaveBeenCalledTimes(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("keeps an incomplete watchdog subtree as concrete retry debt", async () => {
|
||||
const f = fixture(); f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.incomplete = true;
|
||||
await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
const loaded = f.load.mock.calls.length; f.state.incomplete = false;
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.check).toHaveBeenCalledTimes(2); expect(f.load).toHaveBeenCalledTimes(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("keeps a concurrent producer dirty until the in-flight owner has completed", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); await f.worker.ready;
|
||||
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
|
||||
f.check.mockImplementationOnce(() => held);
|
||||
await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
expect(f.check.mock.calls.length).toBeGreaterThan(3);
|
||||
});
|
||||
it("retains startup work on standby and cancels future deadlines on shutdown", async () => {
|
||||
const f = fixture(); f.state.standby = true; f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" });
|
||||
await f.worker.ready; await vi.advanceTimersByTimeAsync(90_000); expect(f.load).not.toHaveBeenCalled();
|
||||
f.state.standby = false; f.state.deadline = Date.now() + 60_000; f.worker.wake(); await f.settle();
|
||||
expect(f.check).toHaveBeenCalledOnce(); await f.worker.stop();
|
||||
await vi.advanceTimersByTimeAsync(120_000); expect(f.check).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -41,7 +41,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
||||
beforeAll(async () => {
|
||||
if (externalUrl) {
|
||||
await applyPendingMigrations(externalUrl);
|
||||
db = createDb(externalUrl);
|
||||
db = createDb(externalUrl, { maxConnections: 6 });
|
||||
return;
|
||||
}
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-");
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createServer } from "node:http";
|
||||
import { createS3StorageProvider } from "../storage/s3-provider.js";
|
||||
import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js";
|
||||
import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
@@ -77,6 +79,93 @@ describe.skipIf(!support.supported)("managed skills use committed versions", ()
|
||||
.toMatchObject({ content: "complete root guide" });
|
||||
});
|
||||
|
||||
it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => {
|
||||
const companyId = await company();
|
||||
const objects = new Map<string, Buffer>();
|
||||
const server = createServer((request, response) => {
|
||||
void (async () => {
|
||||
const key = new URL(request.url!, "http://fixture").pathname;
|
||||
if (request.method === "PUT") {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) chunks.push(Buffer.from(chunk));
|
||||
let body = Buffer.concat(chunks);
|
||||
if (request.headers["content-encoding"]?.includes("aws-chunked")) {
|
||||
const decoded: Buffer[] = [];
|
||||
let offset = 0;
|
||||
while (offset < body.length) {
|
||||
const end = body.indexOf("\r\n", offset);
|
||||
const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16);
|
||||
if (size === 0) break;
|
||||
decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2;
|
||||
}
|
||||
body = Buffer.concat(decoded);
|
||||
}
|
||||
objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return;
|
||||
}
|
||||
const body = objects.get(key);
|
||||
if (!body) { response.writeHead(404).end(); return; }
|
||||
response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' });
|
||||
response.end(request.method === "HEAD" ? undefined : body);
|
||||
})().catch(() => { response.writeHead(500).end(); });
|
||||
});
|
||||
await new Promise<void>(resolve => { server.listen(0, "127.0.0.1", resolve); });
|
||||
const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address");
|
||||
const endpoint = `http://127.0.0.1:${address.port}`;
|
||||
const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" };
|
||||
const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]]));
|
||||
Object.assign(process.env, credentials);
|
||||
vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({
|
||||
bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true,
|
||||
})));
|
||||
const key = "cloud-runtime-config:v1";
|
||||
try {
|
||||
await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" });
|
||||
const run = (phase: string) => promisify(execFile)(process.execPath, [
|
||||
"--import", fileURLToPath(import.meta.resolve("tsx")),
|
||||
fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)),
|
||||
`--phase=${phase}`, `--company=${companyId}`, "--limit=50",
|
||||
], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString,
|
||||
PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture",
|
||||
PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3",
|
||||
PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1",
|
||||
PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" },
|
||||
timeout: 60_000, maxBuffer: 1024 * 1024 });
|
||||
await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration");
|
||||
expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0);
|
||||
await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } });
|
||||
const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]);
|
||||
const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const);
|
||||
const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]);
|
||||
expect([...objects.entries()]).toEqual(before);
|
||||
const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!;
|
||||
const archive = objects.get(archiveKey)!;
|
||||
const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1;
|
||||
objects.set(archiveKey, corrupted);
|
||||
const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]);
|
||||
objects.set(archiveKey, archive);
|
||||
const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!;
|
||||
const manifest = objects.get(manifestKey)!; objects.delete(manifestKey);
|
||||
const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]);
|
||||
objects.set(manifestKey, manifest);
|
||||
const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]);
|
||||
expect([...objects.entries()].sort()).toEqual([...before].sort());
|
||||
expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key)))
|
||||
.toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]);
|
||||
} finally {
|
||||
await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key));
|
||||
vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))));
|
||||
for (const [key, value] of Object.entries(previousCredentials)) {
|
||||
if (value === undefined) delete process.env[key]; else process.env[key] = value;
|
||||
}
|
||||
server.closeAllConnections(); await new Promise<void>(resolve => { server.close(() => resolve()); });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
it.each([
|
||||
{ storage: "legacy local", env: { PAPERCLIP_STORAGE_PROVIDER: "local_disk" }, expectedStatus: "local_storage" },
|
||||
{ storage: "shared S3", env: { PAPERCLIP_STORAGE_PROVIDER: "s3",
|
||||
|
||||
@@ -135,6 +135,9 @@ const {
|
||||
})),
|
||||
reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })),
|
||||
reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })),
|
||||
registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerRetainedNativeSessionCleanupWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve(), stop: vi.fn() })),
|
||||
@@ -953,6 +956,21 @@ describe("startServer feedback export wiring", () => {
|
||||
if (enabled) expect(heartbeatServiceMock.registerCapturedInstructionCommitWork.mock.calls[0][1]()).toBe(true);
|
||||
});
|
||||
|
||||
it.each([false, true])("registers retry and queued dispatch only when maintenance is enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled();
|
||||
if (enabled) {
|
||||
const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0];
|
||||
const app = await createAppMock.mock.results.at(-1)!.value;
|
||||
expect(coordinator).toBe(app.locals.deliveryWork);
|
||||
expect(admitted()).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])("registers retry cancellation recovery only when maintenance is enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
@@ -1351,13 +1369,14 @@ describe("startServer feedback export wiring", () => {
|
||||
await startServer();
|
||||
await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
|
||||
expect(intervalCallback).not.toBeNull();
|
||||
intervalCallback?.();
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2);
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled();
|
||||
expect(retiredDetector).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
delete (runtime as Partial<typeof runtime>).reconcileProductivityReviews;
|
||||
|
||||
Reference in new issue
Block a user