import * as wakeSettlementNotifications from "../services/external-conversation-settlement-notifications.js"; import { heartbeatService } from "../services/heartbeat.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js"; import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js"; import { statusDecisionEffects } from "@paperclipai/db"; import { randomUUID } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { agentWakeupRequests, agents, companies, completionContracts, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations, type Db } from "@paperclipai/db"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { nativeRunProjectionWork } from "../services/native-runtime/native-run-projection-work.js"; import { projectCommittedRun } from "../services/native-runtime/native-run-projection.js"; import { NATIVE_RUN_FAILED_UNEXPLAINED } from "../services/native-runtime/native-run-failure-explanation.js"; import { notifyNativeFinalizationWork } from "../services/native-finalization-notifications.js"; import { finalizeNativeRun, recordNativeFinalizationFailure } from "../services/native-runtime/native-run-finalizer.js"; import * as statusCommitter from "../services/native-runtime/status-decision-committer.js"; import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js"; import { workspaceOperationService } from "../services/workspace-operations.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../vendor/paperclip-runner/testing.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { emitAgentTaskRun } from "../services/agent-task-run-telemetry.js"; vi.mock("../services/agent-task-run-telemetry.js", () => ({ emitAgentTaskRun: vi.fn(async () => undefined) })); vi.mock("../services/run-failure-report.js", async importOriginal => ({ ...await importOriginal(), reportRunFailure: vi.fn(async () => undefined), })); describe("committed native run projection lifecycle", () => { let temporary: Awaited> | undefined; let db: Db, root: string; let coordinator: ReturnType | undefined; let worker: ReturnType["register"]> | undefined; const companyId = randomUUID(), agentId = randomUUID(); const priorLogs = process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; let admitted = true; const errors: unknown[] = []; beforeAll(async () => { root = await mkdtemp(join(tmpdir(), "native-run-projection-")); process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = root; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_RUN_PROJECTION_TEST_DATABASE_URL"); if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-run-projection-"); db = createDb(external ?? temporary!.connectionString); await db.insert(companies).values({ id: companyId, name: "Result commits", issuePrefix: `RC${randomUUID().slice(0, 6)}` }); await db.insert(agents).values({ id: agentId, companyId, name: "Result worker", adapterType: "paperclip_runner" }); }, 30_000); beforeEach(() => { admitted = true; errors.length = 0; vi.mocked(emitAgentTaskRun).mockClear(); vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); }); afterEach(async () => { await coordinator?.stop(); coordinator = undefined; worker = undefined; vi.useRealTimers(); vi.restoreAllMocks(); await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, companyId)); expect(errors).toEqual([]); }); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); await rm(root, { recursive: true, force: true }); if (priorLogs === undefined) delete process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; else process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = priorLogs; }); async function start(project = projectCommittedRun) { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, canReconcile: () => admitted, onError: error => errors.push(error) }); worker = nativeRunProjectionWork(db, project).register(coordinator, () => admitted); await worker.ready; } async function flush() { await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); }, { timeout: 5000 }); } async function quiet() { const previous = db.$client.options.debug; const queries: string[] = []; db.$client.options.debug = (_id, query) => { queries.push(query); }; try { await vi.advanceTimersByTimeAsync(120_000); await delay(10); expect(queries).toEqual([]); } finally { db.$client.options.debug = previous; } } async function seed(matchesCompletionClaim = false) { const runId = randomUUID(), issueId = randomUUID(), contractId = randomUUID(), sourceInstanceId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Commit accepted work", status: "in_progress", assigneeAgentId: agentId }); await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "result-worker-v1", risk: matchesCompletionClaim ? "low" : "standard", completionAuthority: matchesCompletionClaim ? "agent_claim_policy" : "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: matchesCompletionClaim ? CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision : "result-worker-v1", objective: "Finish", criteria: [{ id: "objective", requirement: "Complete the task" }] }, canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" }); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, runtimeMode: "native", status: "running", nativeIssueId: issueId, nativeSessionId: runId, runnerInstanceId: sourceInstanceId, completionContractId: contractId, completionContractSha256: contractId, contextSnapshot: { issueId } }); await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); return { companyId, issueId, runId, agentId, sessionId: runId, completionContractId: contractId, completionContractSha256: contractId, sourceInstanceId, controlPlaneSourceInstanceId: "result-worker-test" }; } async function complete(binding: Awaited>, producer: "control_plane" | "coordinator", writer: Db = db, terminalStatus: "succeeded" | "failed" | "cancelled" = "succeeded") { const port = producer === "control_plane" ? new PaperclipControlPlanePort(writer, binding) : new NativeRunCoordinatorStore(writer, { ...binding, normalizedSessionId: binding.sessionId, runnerSourceInstanceId: binding.sourceInstanceId, completionContractRevision: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision, completionContractCriterionIds: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.criteria.map(item => item.criterionId) }); await port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, runTerminalState: terminalStatus }, callerResultId: `${binding.runId}:result` }); } async function barrier(binding: Awaited>) { await workspaceOperationService(db).createRecorder({ companyId, issueId: binding.issueId, heartbeatRunId: binding.runId }) .recordOperation({ phase: "workspace_finalize", cwd: root, run: async () => ({ status: "succeeded", exitCode: 0 }) }); } async function state(runId: string) { return db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]); } async function read(runId: string) { return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).then(rows => rows[0]); } async function accepted(status: "succeeded" | "failed" | "cancelled" = "succeeded") { const binding = await seed(); await complete(binding, "control_plane", db, status); await barrier(binding); return binding; } async function claimedWake(binding: Awaited>) { const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "issue", status: "claimed", runId: binding.runId, claimedAt: new Date(), payload: { issueId: binding.issueId } }).returning(); await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, binding.runId)); return wake; } const readWake = async (id: string) => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, id)))[0]; it.each(["succeeded", "failed"] as const)("preserves actual operator Stop when accepted %s work finishes delayed workspace finalization", async terminalStatus => { const binding = await seed(true); await complete(binding, "control_plane", db, terminalStatus); const wake = await claimedWake(binding); let entered!: () => void, release!: () => void; const started = new Promise(resolve => { entered = resolve; }); const gate = new Promise(resolve => { release = resolve; }); const finalizing = (async () => { await workspaceOperationService(db).createRecorder({ companyId, issueId: binding.issueId, heartbeatRunId: binding.runId }) .recordOperation({ phase: "workspace_finalize", cwd: root, run: async () => { entered(); await gate; return { status: "succeeded", exitCode: 0 }; } }); return finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); })(); let stopped: Awaited> | undefined; let stoppedIssue: typeof issues.$inferSelect | undefined; try { await started; await heartbeatService(db).cancelRun(binding.runId, "Stopped by operator", { cancellationRequestId: randomUUID(), cancellationRequestedByUserId: "board-user", resultJson: { cancelledByActorType: "user", cancelledByUserId: "board-user" }, }); stopped = await read(binding.runId); [stoppedIssue] = await db.select().from(issues).where(eq(issues.id, binding.issueId)); expect(["in_progress", "blocked"]).toContain(stoppedIssue.status); expect(stopped.status).toBe("cancelled"); expect(stopped.resultJson?.nativeCancellation).toMatchObject({ scope: "run", dispatched: false, dispatchState: "acknowledged" }); expect(await readWake(wake.id)).toMatchObject({ status: "cancelled" }); } finally { release(); await finalizing; } const after = await read(binding.runId); expect(after).toMatchObject({ status: "cancelled", finishedAt: stopped!.finishedAt, errorCode: stopped!.errorCode }); expect(after.resultJson?.nativeCancellation).toEqual(stopped!.resultJson?.nativeCancellation); expect((await state(binding.runId)).resultId).not.toBeNull(); // Run-only Stop does not revoke the server arbiter's already accepted // result. Task completion stays separate from the cancelled execution. expect((await db.select().from(issues).where(eq(issues.id, binding.issueId)))[0].status) .toBe(terminalStatus === "succeeded" ? "done" : "blocked"); expect(after).toMatchObject({ costAccountingPending: stopped!.costAccountingPending, costAccountedAt: stopped!.costAccountedAt, usageJson: stopped!.usageJson }); expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.nativeIssueId, binding.issueId))) .toEqual([{ id: binding.runId }]); await start(); await flush(); await quiet(); expect((await read(binding.runId)).status).toBe("cancelled"); expect(await readWake(wake.id)).toMatchObject({ status: "cancelled" }); }); it("preserves Stop that wins after the accepted task decision but before direct run projection", async () => { const binding = await seed(true); await complete(binding, "control_plane"); await barrier(binding); const wake = await claimedWake(binding); let stopped: Awaited> | undefined; let stoppedIssue: typeof issues.$inferSelect | undefined; const commit = statusCommitter.commitNativeStatusDecision; vi.spyOn(statusCommitter, "commitNativeStatusDecision").mockImplementationOnce(async input => { const decision = await commit(input); await heartbeatService(db).cancelRun(binding.runId, "Stopped by operator", { cancellationRequestId: randomUUID(), cancellationRequestedByUserId: "board-user", resultJson: { cancelledByActorType: "user", cancelledByUserId: "board-user" }, }); stopped = await read(binding.runId); [stoppedIssue] = await db.select().from(issues).where(eq(issues.id, binding.issueId)); expect(stopped.status).toBe("cancelled"); return decision; }); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const after = await read(binding.runId); expect(after).toMatchObject({ status: "cancelled", finishedAt: stopped!.finishedAt, error: stopped!.error, errorCode: stopped!.errorCode }); expect(after.resultJson?.nativeCancellation).toEqual(stopped!.resultJson?.nativeCancellation); expect(await readWake(wake.id)).toMatchObject({ status: "cancelled", finishedAt: stopped!.finishedAt }); // The task decision committed first. Run-only Stop preserves that decision; // it cannot reopen completed work or erase its accepted result. expect(stoppedIssue!.status).toBe("done"); expect((await db.select().from(issues).where(eq(issues.id, binding.issueId)))[0]).toMatchObject({ status: stoppedIssue!.status, statusVersion: stoppedIssue!.statusVersion, lastStatusDecisionId: stoppedIssue!.lastStatusDecisionId, }); await projectCommittedRun({ db, run: after, coordinator: await state(binding.runId) }); expect(await read(binding.runId)).toEqual(after); }); it.each(["succeeded", "failed", "cancelled"] as const)("settles the exact wake atomically with a fresh %s projection", async status => { const binding = await accepted(status), wake = await claimedWake(binding); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const run = await read(binding.runId), settled = await readWake(wake.id); expect(run.status).toBe(status); expect(settled).toMatchObject({ status: status === "succeeded" ? "completed" : status, runId: binding.runId, companyId, agentId, finishedAt: run.finishedAt, error: run.error }); expect(settled.finishedAt).not.toBeNull(); await projectCommittedRun({ db, run, coordinator: await state(binding.runId) }); expect(await readWake(wake.id)).toEqual(settled); }); it.each(["provider", "workspace"] as const)("preserves actual Stop through %s finalization retry exhaustion", async scope => { const binding = await seed(true); await complete(binding, "control_plane"); const wake = await claimedWake(binding); await workspaceOperationService(db).createRecorder({ companyId, issueId: binding.issueId, heartbeatRunId: binding.runId }) .recordOperation({ phase: "workspace_finalize", cwd: root, run: async () => ({ status: scope === "workspace" ? "failed" : "succeeded", exitCode: scope === "workspace" ? 1 : 0 }) }); await heartbeatService(db).cancelRun(binding.runId, "Stopped by operator", { cancellationRequestId: randomUUID(), cancellationRequestedByUserId: "board-user", resultJson: { cancelledByActorType: "user", cancelledByUserId: "board-user" }, }); const stopped = await read(binding.runId), cancelledWake = await readWake(wake.id); expect(stopped.status).toBe("cancelled"); expect(stopped.resultJson?.nativeCancellation).toMatchObject({ scope: "run", dispatched: false, dispatchState: "acknowledged" }); const acceptedResult = (await state(binding.runId)).resultId; for (let attempt = 0; attempt < 3; attempt++) { if (scope === "workspace") await recordNativeFinalizationFailure({ db, runId: binding.runId, projectRunStatus: true, failureScope: "workspace", error: new Error("native_workspace_finalize_failed") }); else await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, failpoint: "status_projection" }); } const after = await read(binding.runId); expect((await state(binding.runId))).toMatchObject({ phase: "terminal_failure", resultId: acceptedResult, nextAttemptAt: null }); expect(after).toMatchObject({ status: "cancelled", finishedAt: stopped.finishedAt, error: stopped.error, errorCode: stopped.errorCode, costAccountingPending: stopped.costAccountingPending, costAccountedAt: stopped.costAccountedAt, usageJson: stopped.usageJson }); expect(after.resultJson?.nativeCancellation).toEqual(stopped.resultJson?.nativeCancellation); expect(await readWake(wake.id)).toEqual(cancelledWake); expect(after.resultJson).toMatchObject({ finalizationPhase: "terminal_failure", nextAttemptAt: null }); expect((await db.select().from(issues).where(eq(issues.id, binding.issueId)))[0].status) .toBe(scope === "workspace" ? "blocked" : "in_review"); await start(); await flush(); await quiet(); expect((await read(binding.runId)).status).toBe("cancelled"); }); it("preserves Stop that wins after the workspace terminal-failure admission snapshot", async () => { const binding = await seed(true); await complete(binding, "control_plane"); const wake = await claimedWake(binding); await workspaceOperationService(db).createRecorder({ companyId, issueId: binding.issueId, heartbeatRunId: binding.runId }) .recordOperation({ phase: "workspace_finalize", cwd: root, run: async () => ({ status: "failed", exitCode: 1 }) }); const fail = () => recordNativeFinalizationFailure({ db, runId: binding.runId, projectRunStatus: true, failureScope: "workspace" as const, permanent: true, error: new Error("native_workspace_finalize_failed") }); let stopped: Awaited> | undefined; let cancelledWake: Awaited> | undefined; const transaction = db.transaction.bind(db); const stopAfterSnapshot: typeof db.transaction = async (callback, config) => { // recordNativeFinalizationFailure has already read its run/coordinator. // Let the actual Stop producer win before its failure transaction starts. expect((await read(binding.runId)).status).toBe("running"); await heartbeatService(db).cancelRun(binding.runId, "Stopped by operator", { cancellationRequestId: randomUUID(), cancellationRequestedByUserId: "board-user", resultJson: { cancelledByActorType: "user", cancelledByUserId: "board-user" }, }); stopped = await read(binding.runId); cancelledWake = await readWake(wake.id); expect(stopped.status).toBe("cancelled"); return transaction(callback, config); }; vi.spyOn(db, "transaction").mockImplementationOnce(stopAfterSnapshot); await fail(); const after = await read(binding.runId); expect(after).toMatchObject({ status: "cancelled", finishedAt: stopped!.finishedAt, error: stopped!.error, errorCode: stopped!.errorCode }); expect(after.resultJson?.nativeCancellation).toEqual(stopped!.resultJson?.nativeCancellation); expect(await readWake(wake.id)).toEqual(cancelledWake); expect((await state(binding.runId)).phase).toBe("terminal_failure"); }); it.each(["provider", "workspace"] as const)("settles the exact wake when %s finalization retries exhaust", async scope => { const binding = await accepted(), wake = await claimedWake(binding); if (scope === "workspace") await db.update(workspaceOperations).set({ status: "failed" }) .where(eq(workspaceOperations.heartbeatRunId, binding.runId)); if (scope === "provider") await db.update(nativeRunResults).set({ resultJson: sql`jsonb_set(${nativeRunResults.resultJson}, '{terminal,runTerminalState}', '"unknown"'::jsonb)`, }).where(eq(nativeRunResults.runId, binding.runId)); for (let attempt = 0; attempt < 3; attempt++) { await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) }) .where(eq(nativeRunFinalizations.runId, binding.runId)); if (scope === "provider") await reconcileNativeFinalizations(db, [binding.runId]); else await recordNativeFinalizationFailure({ db, runId: binding.runId, projectRunStatus: true, failureScope: "workspace", error: new Error("native_workspace_finalize_failed") }); if (attempt < 2) { expect((await read(binding.runId)).status).toBe("running"); expect(await readWake(wake.id)).toMatchObject({ status: "claimed", finishedAt: null }); } } const run = await read(binding.runId); expect((await state(binding.runId)).phase).toBe("terminal_failure"); expect(run).toMatchObject({ status: scope === "provider" ? "succeeded" : "failed", nativePhase: "terminal_failure" }); expect(run.finishedAt).not.toBeNull(); expect(await readWake(wake.id)).toMatchObject({ status: scope === "provider" ? "completed" : "failed", runId: binding.runId, finishedAt: run.finishedAt }); }); it("keeps the source wake claimed when exhausted failure does not own terminal projection", async () => { const binding = await accepted(), wake = await claimedWake(binding); await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.heartbeatRunId, binding.runId)); for (let attempt = 0; attempt < 3; attempt++) await recordNativeFinalizationFailure({ db, runId: binding.runId, projectRunStatus: false, failureScope: "workspace", error: new Error("native_workspace_finalize_failed") }); expect((await state(binding.runId)).phase).toBe("terminal_failure"); expect((await read(binding.runId)).status).toBe("running"); expect(await readWake(wake.id)).toEqual(wake); }); it("rolls back exhausted projection and its claim if wake notification registration fails", async () => { const binding = await accepted(), wake = await claimedWake(binding); await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.heartbeatRunId, binding.runId)); const fail = () => recordNativeFinalizationFailure({ db, runId: binding.runId, projectRunStatus: true, failureScope: "workspace" as const, error: new Error("native_workspace_finalize_failed") }); await fail(); await fail(); const beforeRun = await read(binding.runId), beforeCoordinator = await state(binding.runId); const notify = wakeSettlementNotifications.notifyExternalConversationSettlement; vi.spyOn(wakeSettlementNotifications, "notifyExternalConversationSettlement").mockImplementation(async (tx, scope) => { const [current] = await (tx as Db).select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); expect(current.status).toBe("claimed"); await notify(tx, scope); throw new Error("exhaustion wake notification failure"); }); await expect(fail()).rejects.toThrow("exhaustion wake notification failure"); expect(await read(binding.runId)).toEqual(beforeRun); expect(await state(binding.runId)).toEqual(beforeCoordinator); expect(await readWake(wake.id)).toEqual(wake); }); it.each(["succeeded", "failed", "cancelled"] as const)("discovers and repairs an old %s run's claimed wake once", async status => { const binding = await accepted(status); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const run = await read(binding.runId), wake = await claimedWake(binding); const work = nativeRunProjectionWork(db); const [entry] = await work.load({ companyId, runId: binding.runId }); expect(entry).toMatchObject({ id: binding.runId }); await work.check(entry); expect(await readWake(wake.id)).toMatchObject({ status: status === "succeeded" ? "completed" : status, finishedAt: run.finishedAt }); expect((await read(binding.runId)).finishedAt).toEqual(run.finishedAt); expect(await work.load({ companyId, runId: binding.runId })).toEqual([]); }); it.each(["unbound", "newer-run", "queued", "cancelled", "foreign-company", "foreign-agent", "live-owner"] as const)( "does not settle a %s wake claim", async fault => { const binding = await accepted(), wake = await claimedWake(binding); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); if (fault === "unbound" || fault === "newer-run") await db.update(agentWakeupRequests) .set({ runId: fault === "unbound" ? null : randomUUID() }).where(eq(agentWakeupRequests.id, wake.id)); if (fault === "queued" || fault === "cancelled") await db.update(agentWakeupRequests) .set({ status: fault }).where(eq(agentWakeupRequests.id, wake.id)); if (fault === "foreign-company") { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other wake owner", issuePrefix: `OW${randomUUID().slice(0, 6)}` }); await db.update(agentWakeupRequests).set({ companyId: otherCompany }).where(eq(agentWakeupRequests.id, wake.id)); } if (fault === "foreign-agent") { const otherAgent = randomUUID(); await db.insert(agents).values({ id: otherAgent, companyId, name: "Other wake owner", adapterType: "paperclip_runner" }); await db.update(agentWakeupRequests).set({ agentId: otherAgent }).where(eq(agentWakeupRequests.id, wake.id)); } if (fault === "live-owner") await db.update(heartbeatRuns).set({ status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" }).where(eq(heartbeatRuns.id, binding.runId)); const before = await readWake(wake.id); await projectCommittedRun({ db, run: await read(binding.runId), coordinator: await state(binding.runId) }); expect(await readWake(wake.id)).toEqual(before); if (fault === "live-owner") expect((await read(binding.runId)).status).toBe("running"); }, ); it("registers wake settlement intent before mutation and rolls back notification failure", async () => { const binding = await accepted(), wake = await claimedWake(binding); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); const before = await read(binding.runId); const notify = wakeSettlementNotifications.notifyExternalConversationSettlement; const notification = vi.spyOn(wakeSettlementNotifications, "notifyExternalConversationSettlement") .mockImplementation(async (tx, scope) => { const [current] = await (tx as Db).select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); expect(current.status).toBe("claimed"); await notify(tx, scope); throw new Error("wake intent registration failure"); }); await expect(projectCommittedRun({ db, run: before, coordinator: await state(binding.runId) })) .rejects.toThrow("wake intent registration failure"); expect(notification).toHaveBeenCalledOnce(); expect(await read(binding.runId)).toEqual(before); expect(await readWake(wake.id)).toEqual(wake); }); it("rolls back terminal projection and exact wake settlement together", async () => { const binding = await accepted(), wake = await claimedWake(binding); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); const before = await read(binding.runId); await expect(db.transaction(async tx => { await projectCommittedRun({ db: tx as unknown as Db, run: before, coordinator: await state(binding.runId) }); throw new Error("projection rollback fixture"); })).rejects.toThrow("projection rollback fixture"); expect(await read(binding.runId)).toEqual(before); expect(await readWake(wake.id)).toEqual(wake); }); it("reconstructs a committed decision with an unfinished run projection and stays quiet after repair", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); expect((await read(binding.runId)).status).toBe("running"); await start(); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); expect(emitAgentTaskRun).toHaveBeenCalledTimes(1); await quiet(); const settled = await read(binding.runId); await coordinator!.stop(); await start(); await flush(); await quiet(); expect((await read(binding.runId)).executionStatusDeliveryId).toBe(settled.executionStatusDeliveryId); expect(emitAgentTaskRun).toHaveBeenCalledTimes(1); }); it("explains a failed accepted result when the finalizer projects the run", async () => { const binding = await accepted("failed"); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const run = await read(binding.runId); expect(run).toMatchObject({ status: "failed", errorCode: NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, error: NATIVE_RUN_FAILED_UNEXPLAINED.error }); }); it.each([ ["no recorded cause", null, null, NATIVE_RUN_FAILED_UNEXPLAINED.errorCode, NATIVE_RUN_FAILED_UNEXPLAINED.error], ["an empty error", "adapter_failed", " ", "adapter_failed", NATIVE_RUN_FAILED_UNEXPLAINED.error], ["a recorded cause", "adapter_failed", "Adapter exited 1", "adapter_failed", "Adapter exited 1"], ])("explains a failed accepted result in committed recovery projection with %s", async (_label, errorCode, error, expectedCode, expectedError) => { const binding = await accepted("failed"); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); await db.update(heartbeatRuns).set({ errorCode, error }).where(eq(heartbeatRuns.id, binding.runId)); const run = await read(binding.runId); expect(run.status).toBe("running"); await projectCommittedRun({ db, run, coordinator: await state(binding.runId) }); expect(await read(binding.runId)).toMatchObject({ status: "failed", errorCode: expectedCode, error: expectedError }); }); it("ignores routine company-only finalization hints and still repairs a committed run", async () => { await start(); await flush(); await quiet(); const previous = db.$client.options.debug; const queries: string[] = []; db.$client.options.debug = (_id, query) => { if (/from\s+"native_run_finalizations"/i.test(query)) queries.push(query); }; try { for (let index = 0; index < 3; index += 1) { await db.transaction(async tx => { await notifyNativeFinalizationWork(tx, { companyId }); }); await flush(); } expect(queries).toEqual([]); } finally { db.$client.options.debug = previous; } const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); expect(emitAgentTaskRun).toHaveBeenCalledTimes(1); await quiet(); }); it("wakes from the real status decision transaction after startup has become quiet", async () => { await start(); await flush(); await quiet(); const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); await quiet(); }); it.each(["commit", "rollback"])("honors the status writer's outer %s", async outcome => { await start(); await flush(); const binding = await accepted(); const transaction = db.transaction(async tx => { await finalizeNativeRun({ db: tx as unknown as Db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); await vi.advanceTimersByTimeAsync(5000); expect((await read(binding.runId)).status).toBe("running"); if (outcome === "rollback") throw new Error("rollback fixture"); }); if (outcome === "rollback") await expect(transaction).rejects.toThrow("rollback fixture"); else await transaction; await flush(); expect((await read(binding.runId)).status).toBe(outcome === "commit" ? "succeeded" : "running"); await quiet(); }); it("does not emit another terminal event when recovery projects before the live finalizer", async () => { const binding = await accepted(); const commit = statusCommitter.commitNativeStatusDecision; const hook = vi.spyOn(statusCommitter, "commitNativeStatusDecision").mockImplementation(async input => { const outcome = await commit(input); await projectCommittedRun({ db, run: await read(binding.runId), coordinator: await state(binding.runId) }); expect((await read(binding.runId)).status).toBe("succeeded"); return outcome; }); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); expect(hook).toHaveBeenCalled(); expect(emitAgentTaskRun).toHaveBeenCalledTimes(1); }); it("uses fresh completion time when the caller's projection snapshot is stale", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); const stale = await read(binding.runId); const finishedAt = new Date("2020-01-01T00:00:00Z"); await db.update(heartbeatRuns).set({ finishedAt }).where(eq(heartbeatRuns.id, binding.runId)); await projectCommittedRun({ db, run: stale, coordinator: await state(binding.runId) }); expect((await read(binding.runId)).finishedAt).toEqual(finishedAt); }); it.each(["succeeded", "failed", "cancelled"] as const)("stays quiet with retained %s history", async status => { const binding = await accepted(status); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const before = await read(binding.runId); expect(before.status).toBe(status); const calls = vi.mocked(emitAgentTaskRun).mock.calls.length; await start(); await flush(); await quiet(); expect(await read(binding.runId)).toEqual(before); expect(emitAgentTaskRun).toHaveBeenCalledTimes(calls); }); it("serializes competing recovery projections without duplicate terminal events", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); const input = { db, run: await read(binding.runId), coordinator: await state(binding.runId) }; await Promise.all(Array.from({ length: 4 }, () => projectCommittedRun(input))); expect((await read(binding.runId)).status).toBe("succeeded"); expect(emitAgentTaskRun).toHaveBeenCalledTimes(1); }); it("repairs projection metadata and preserves unrelated diagnostics", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const before = await read(binding.runId); await db.update(heartbeatRuns).set({ nativePhase: "retryable_failure", error: "old failure", errorCode: "transport_error", resultJson: { ...before.resultJson, failureCode: "old", nextAttemptAt: "old", unrelatedReceipt: { key: "preserve" } }, }).where(eq(heartbeatRuns.id, binding.runId)); const calls = vi.mocked(emitAgentTaskRun).mock.calls.length; await start(); await flush(); expect(await read(binding.runId)).toMatchObject({ status: "succeeded", nativePhase: "committed", error: null, errorCode: null, finishedAt: before.finishedAt, resultJson: { failureCode: null, nextAttemptAt: null, unrelatedReceipt: { key: "preserve" }, recoveredExecutionFailure: { error: "old failure", errorCode: "transport_error" } } }); expect(emitAgentTaskRun).toHaveBeenCalledTimes(calls); await quiet(); }); it.each(["restore", "worktree"])("respects %s suppression through the public heartbeat registration", async mode => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); const runtimeEnv = mode === "restore" ? { PAPERCLIP_DATABASE_RESTORE_IN_PROGRESS: "true" } : { PAPERCLIP_IN_WORKTREE: "true", PAPERCLIP_INSTANCE_ID: "projection-suppression-test" }; const settings = instanceSettingsService(db, { runtimeEnv }); if (mode === "worktree") await settings.updateExperimental({ enableWorktreeRunExecution: false }); coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, canReconcile: () => admitted, onError: error => errors.push(error) }); worker = heartbeatService(db, { runtimeEnv }).registerNativeRunProjectionWork(coordinator, () => admitted); await worker.ready; await flush(); await quiet(); expect((await read(binding.runId)).status).toBe("running"); if (mode === "worktree") { try { await settings.updateExperimental({ enableWorktreeRunExecution: true }); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); await quiet(); } finally { await settings.updateExperimental({ enableWorktreeRunExecution: false }); } } }); it("retains the first committed decision during drain and projects on resume", async () => { await start(); await flush(); admitted = false; const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); await vi.advanceTimersByTimeAsync(120_000); await quiet(); expect((await read(binding.runId)).status).toBe("running"); admitted = true; await vi.advanceTimersByTimeAsync(30_000); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); await quiet(); }); it("retains a failed projection across drain and restart", async () => { let fail = true; const project: typeof projectCommittedRun = input => { if (fail) { fail = false; throw new Error("projection unavailable"); } return projectCommittedRun(input); }; const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); await start(project); expect(errors).toHaveLength(1); errors.length = 0; expect((await read(binding.runId)).status).toBe("running"); expect(coordinator!.nextWakeAt()).not.toBeNull(); admitted = false; await vi.advanceTimersByTimeAsync(120_000); await quiet(); await coordinator!.stop(); admitted = true; await start(); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); await quiet(); }); it("joins active projection on shutdown and reconstructs later work", async () => { let release!: () => void; const gate = new Promise(resolve => { release = resolve; }); const entered = vi.fn(); const project: typeof projectCommittedRun = async input => { entered(); await gate; return projectCommittedRun(input); }; await start(project); await flush(); const first = await accepted(); await finalizeNativeRun({ db, runId: first.runId, workspaceFinalizeStatus: "succeeded" }); await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(entered).toHaveBeenCalledTimes(1); }); const second = await accepted(); await finalizeNativeRun({ db, runId: second.runId, workspaceFinalizeStatus: "succeeded" }); let stopped = false; const stopping = coordinator!.stop().then(() => { stopped = true; }); await delay(10); expect(stopped).toBe(false); release(); await stopping; expect((await read(first.runId)).status).toBe("succeeded"); expect((await read(second.runId)).status).toBe("running"); await start(); await flush(); expect((await read(second.runId)).status).toBe("succeeded"); await quiet(); }); it("rechecks committed authority after candidate discovery", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); const project: typeof projectCommittedRun = async input => { await db.update(nativeRunFinalizations).set({ phase: "retryable_failure" }).where(eq(nativeRunFinalizations.runId, binding.runId)); return projectCommittedRun(input); }; await start(project); await flush(); expect((await read(binding.runId)).status).toBe("running"); expect(emitAgentTaskRun).not.toHaveBeenCalled(); await quiet(); }); it("waits for real pending-effect settlement before projection", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); const committed = await state(binding.runId); const [effect] = await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.decisionId, committed.decisionId!)); expect(effect).toBeDefined(); await db.update(statusDecisionEffects).set({ deliveryState: "pending", deliveredAt: null }).where(eq(statusDecisionEffects.id, effect.id)); await start(); await flush(); await quiet(); expect((await read(binding.runId)).status).toBe("running"); await statusCommitter.dispatchPendingNativeStatusEffects({ db, ...binding, decisionId: committed.decisionId!, onlyWhenDue: true }); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); await quiet(); }); it("wakes from the real lifecycle status writer when an ownership hold is released", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); await db.update(heartbeatRuns).set({ nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" }) .where(eq(heartbeatRuns.id, binding.runId)); await start(); await flush(); await quiet(); const lifecycle = createHeartbeatLifecycle(db, { emitTerminalAgentTaskRun: vi.fn() } as unknown as HeartbeatLifecycleDependencies); await lifecycle.setRunStatus(binding.runId, "running", { errorCode: null, nativePhase: "committed" }); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); await quiet(); }); it("waits for delegated run projection before committed resource settlement", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); const settled = vi.fn(async () => undefined); await reconcileNativeFinalizations(db, [binding.runId], { skipNativeRunProjection: true, skipNativeResultCommit: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, onWorkspaceSettled: settled }); expect((await read(binding.runId)).status).toBe("running"); expect(settled).not.toHaveBeenCalled(); await start(); await flush(); expect((await read(binding.runId)).status).toBe("succeeded"); await quiet(); await reconcileNativeFinalizations(db, [binding.runId], { skipNativeRunProjection: true, skipNativeResultCommit: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, onWorkspaceSettled: settled }); expect(settled).toHaveBeenCalledTimes(1); }); });