import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { enableRunFailureReporting } from "../../__tests__/helpers/run-failure-reporting.js"; import { randomUUID } from "node:crypto"; import { verifyCommittedReplayPayloadReads } from "./__tests__/helpers/committed-replay.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, completionContracts, createDb, heartbeatRuns, heartbeatRunEvents, issues, nativeRunFinalizations, nativeRunResults, workAssessments, statusDecisions, issueRecoveryActions, agentWakeupRequests, workspaceOperations, activityLog, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "../../__tests__/helpers/embedded-postgres.js"; import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL, } from "../../vendor/paperclip-runner/testing.js"; const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn(), hashPrivateRef: vi.fn((value: string) => `hashed:${value}`), })); vi.mock("../../telemetry.ts", () => ({ getTelemetryClient: () => mockTelemetryClient })); const reconcileMuseAssignments = vi.hoisted(() => vi.fn(async () => undefined)); vi.mock("../muse-runner-broker.js", async importOriginal => ({ ...await importOriginal(), museRunnerBroker: () => ({ reconcileTerminalAssignments: reconcileMuseAssignments }), })); const mockCaptureRunFailure = vi.hoisted(() => vi.fn()); vi.mock("../../sentry.ts", async () => { const actual = await vi.importActual("../../sentry.ts"); return { ...actual, captureRunFailure: mockCaptureRunFailure, }; }); function agentTaskRunCalls(fromIndex: number) { return mockTelemetryClient.track.mock.calls .slice(fromIndex) .filter((call) => call[0] === "agent.task_run"); } function captureRunFailureCallsFrom(fromIndex: number) { return mockCaptureRunFailure.mock.calls.slice(fromIndex); } import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { restoreNativeWorkspaceBestEffort } from "./native-workspace-best-effort.js"; import { reconcileNativeFinalizations } from "./native-finalization-reconciler.js"; import { recoverLegacyUnsafeWorkspaceExports } from "./native-workspace-export-recovery.js"; import { finalizeNativeRun, recordNativeFinalizationFailure } from "./native-run-finalizer.js"; import { waitForPendingRunFailureReports } from "../run-failure-report.js"; import { deliverExecutionStatuses } from "../execution-status-delivery.js"; import { commitNativeStatusDecision } from "./status-decision-committer.js"; import { NATIVE_STATUS_ARBITER_POLICY_VERSION, type NativeStatusDecision } from "./status-arbiter.js"; import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../delivery-work-notifications.js"; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_FINALIZER_TEST_DATABASE_URL"); const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres native-finalizer telemetry tests on this host: ${ embeddedPostgresSupport.reason ?? "unsupported environment" }`, ); } describeEmbeddedPostgres("native run finalizer / status decision committer — agent.task_run emission", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; let companyId: string; let agentId: string; beforeAll(async () => { if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-native-finalizer-telemetry-"); db = createDb(external ?? tempDb!.connectionString); companyId = randomUUID(); agentId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Telemetry", issuePrefix: "TLM" }); await db.insert(agents).values({ id: agentId, companyId, name: "Native worker", adapterType: "codex_local", status: "running", }); }, 30_000); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); async function seedNativeRun() { const issueId = randomUUID(); const contractId = randomUUID(); const runId = randomUUID(); const sessionId = randomUUID(); const runnerInstanceId = randomUUID(); const contractSha256 = `contract-${contractId}`; await db.insert(issues).values({ id: issueId, companyId, title: "Native telemetry fixture", status: "in_progress", assigneeAgentId: agentId, workMode: "standard", }); await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "telemetry-v1", risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "telemetry-v1", objective: "Emit telemetry", criteria: [{ id: "objective", requirement: "Complete the task" }], }, canonicalSha256: contractSha256, createdByActorType: "system", createdByActorId: "test", }); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, nativeSessionId: sessionId, runnerInstanceId, completionContractId: contractId, completionContractSha256: contractSha256, contextSnapshot: { issueId }, }); return { issueId, contractId, contractSha256, runId, sessionId, runnerInstanceId }; } function newPort(fixture: Awaited>) { return new PaperclipControlPlanePort(db, { companyId, issueId: fixture.issueId, runId: fixture.runId, agentId, sessionId: fixture.sessionId, completionContractId: fixture.contractId, completionContractSha256: fixture.contractSha256, sourceInstanceId: fixture.runnerInstanceId, controlPlaneSourceInstanceId: "telemetry-test", }); } async function driveToCompleteResult( fixture: Awaited>, terminal: typeof CONTROL_PLANE_CONFORMANCE_TERMINAL = CONTROL_PLANE_CONFORMANCE_TERMINAL, result = CONTROL_PLANE_CONFORMANCE_RESULT, ) { const port = newPort(fixture); await port.openRun({ identity: { runId: fixture.runId, sessionId: fixture.sessionId, companyId, issueId: fixture.issueId, agentId, }, backendKind: "mock", sourceInstanceId: fixture.runnerInstanceId, }); await port.completeRun({ result, terminal, callerResultId: `${fixture.runId}:result`, }); } it("finishes an accepted result after shutdown failed and workspace repair succeeded", async () => { const fixture = await seedNativeRun(); await db.update(completionContracts).set({ risk: "low", completionAuthority: "agent_claim_policy", contractJson: { revision: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision, objective: "Finish the work", criteria: [{ id: "objective", requirement: "Complete the task" }] } }) .where(eq(completionContracts.id, fixture.contractId)); await driveToCompleteResult(fixture); await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), errorCode: "provider_transport_failed", error: "provider_transport_failed: runner did not durably suspend before checkpoint" }) .where(eq(heartbeatRuns.id, fixture.runId)); await db.update(nativeRunFinalizations).set({ phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: null }).where(eq(nativeRunFinalizations.runId, fixture.runId)); const failed = await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "failed" }); // An unchanged failed workspace must replay its existing decision. await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "failed" }); expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, fixture.runId))).toHaveLength(1); await db.insert(workspaceOperations).values({ companyId, issueId: fixture.issueId, heartbeatRunId: fixture.runId, phase: "workspace_finalize", status: "succeeded" }); await db.update(nativeRunFinalizations).set({ nextAttemptAt: null }).where(eq(nativeRunFinalizations.runId, fixture.runId)); await reconcileNativeFinalizations(db, [fixture.runId]); expect((await db.select().from(issues).where(eq(issues.id, fixture.issueId)))[0]).toMatchObject({ status: "done", executionRunId: null, checkoutRunId: null, }); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)))[0]).toMatchObject({ status: "succeeded", errorCode: null, error: null, resultJson: { recoveredExecutionFailure: { errorCode: "provider_transport_failed" } }, }); const assessments = await db.select().from(workAssessments).where(eq(workAssessments.runId, fixture.runId)); expect(assessments).toHaveLength(2); expect(assessments.find(row => row.id !== failed.assessmentId)?.supersedesAssessmentId).toBe(failed.assessmentId); await reconcileNativeFinalizations(db, [fixture.runId]); expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, fixture.runId))).toHaveLength(2); expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, fixture.runId))).toHaveLength(1); expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).toHaveLength(0); }); it("emits exactly one agent.task_run event when the native finalizer commits a terminal status", async () => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture); const callsBefore = mockTelemetryClient.track.mock.calls.length; const captureCallsBefore = mockCaptureRunFailure.mock.calls.length; await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); const newCalls = agentTaskRunCalls(callsBefore); expect(newCalls).toHaveLength(1); expect(newCalls[0]?.[1]).toMatchObject({ agent_id: agentId, state: "succeeded" }); const run = await db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)) .then((rows) => rows[0]); expect(run?.status).toBe("succeeded"); // "succeeded" is not a failure status, so it never reports to Sentry. expect(captureRunFailureCallsFrom(captureCallsBefore)).toHaveLength(0); }); it("emits zero events when a repeat finalize call preserves the succeeded terminal state", async () => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); await verifyCommittedReplayPayloadReads(db, { runId: fixture.runId, companyId, agentId }, async () => { const callsBefore = mockTelemetryClient.track.mock.calls.length; reconcileMuseAssignments.mockClear(); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); expect(agentTaskRunCalls(callsBefore)).toHaveLength(0); expect(reconcileMuseAssignments).toHaveBeenCalledExactlyOnceWith(companyId, agentId, fixture.runId); await db.insert(workspaceOperations).values({ companyId, issueId: fixture.issueId, heartbeatRunId: fixture.runId, phase: "workspace_finalize", status: "succeeded" }); const reconciled = await reconcileNativeFinalizations(db, [fixture.runId]); expect(reconciled).toContainEqual(expect.objectContaining({ runId: fixture.runId, phase: "committed" })); expect(agentTaskRunCalls(callsBefore)).toHaveLength(0); }); }); it("repairs a stale succeeded-run error once without duplicate telemetry or destroying retained owner evidence", async () => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); await db .update(heartbeatRuns) .set({ errorCode: "adapter_failed", error: "provider_transport_failed: retained cleanup evidence", processPid: 987654, runnerProfileJson: { sessionCheckpoint: { retainedEvidence: "keep" } }, }) .where(eq(heartbeatRuns.id, fixture.runId)); const callsBefore = mockTelemetryClient.track.mock.calls.length; await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); const [recovered] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)); expect(recovered).toMatchObject({ status: "succeeded", error: null, errorCode: null, processPid: 987654, runnerProfileJson: { sessionCheckpoint: { retainedEvidence: "keep" } }, resultJson: { recoveredExecutionFailure: { schema: "paperclip.recovered_execution_failure.v1", errorCode: "adapter_failed", error: "provider_transport_failed: retained cleanup evidence", }, }, }); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); const [replayed] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)); expect(replayed.resultJson?.recoveredExecutionFailure).toEqual( recovered.resultJson?.recoveredExecutionFailure, ); expect(agentTaskRunCalls(callsBefore)).toHaveLength(0); }); it.each([ "native_execution_ownership_unverified", "native_adopted_runner_authentication_timeout", ])( "does not project through retained ownership guard %s", async (errorCode) => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); const [held] = await db .update(heartbeatRuns) .set({ status: "running", finishedAt: null, nativePhase: "terminal_failure", errorCode, error: "Retained owner must remain fenced", processPid: 987654, }) .where(eq(heartbeatRuns.id, fixture.runId)) .returning(); const callsBefore = mockTelemetryClient.track.mock.calls.length; await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); const [after] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)); expect(after).toEqual(held); expect(agentTaskRunCalls(callsBefore)).toHaveLength(0); }, ); it.each([null, "Earlier diagnostic"])( "preserves the current-row diagnostic when it changes after finalizer admission (prior=%s)", async (priorError) => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); await db .update(heartbeatRuns) .set({ error: priorError, errorCode: priorError ? "adapter_failed" : null, }) .where(eq(heartbeatRuns.id, fixture.runId)); const select = db.select.bind(db); let changed = false; // Hold the persisted-result read inside projectCommittedRun, after its // caller captured the run but before the terminal UPDATE. This models // a late heartbeat cleanup diagnostic without changing production APIs. const wrap = (query: any): any => new Proxy(query, { get(target, key) { const value = Reflect.get(target, key, target); if (key === "then") return async (fulfilled: any, rejected: any) => { if (!changed) { changed = true; await db .update(heartbeatRuns) .set({ error: "Latest cleanup diagnostic", errorCode: "provider_transport_failed", resultJson: { concurrentMarker: "keep", nativeCommittedChatResponse: { resultId: "retain-selector", }, }, }) .where(eq(heartbeatRuns.id, fixture.runId)); } return value.call(target, fulfilled, rejected); }; return typeof value === "function" ? (...args: any[]) => wrap(value.apply(target, args)) : value; }, }); const spy = vi.spyOn(db, "select").mockImplementation((( selection: any, ) => { const query = select(selection); return selection?.terminalState ? wrap(query) : query; }) as typeof db.select); try { await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); } finally { spy.mockRestore(); } expect(changed).toBe(true); const [after] = await db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)); expect(after).toMatchObject({ error: null, errorCode: null, resultJson: { concurrentMarker: "keep", nativeCommittedChatResponse: { resultId: "retain-selector" }, recoveredExecutionFailure: { error: "Latest cleanup diagnostic", errorCode: "provider_transport_failed", }, }, }); }, ); it.each(["in_progress", "cancelled"])("does not redeliver a committed failure during reconciliation of a %s task", async (issueStatus) => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture, { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, turnTerminalState: "failed", runTerminalState: "failed", }); // workspaceFinalizeStatus reports whether the workspace finalization // step itself succeeded, independent of the run's own terminal state // (runTerminalState below), which is what actually failed here. const captureCallsBeforeFirstFinalize = mockCaptureRunFailure.mock.calls.length; await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }); const run = await db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)) .then((rows) => rows[0]); expect(run?.status).toBe("failed"); // The first write is a genuine transition into "failed": it reports // exactly one Sentry event. await vi.waitFor(() => { expect(captureRunFailureCallsFrom(captureCallsBeforeFirstFinalize)).toHaveLength(1); }, { timeout: 5_000 }); const firstFinalizeCaptures = captureRunFailureCallsFrom(captureCallsBeforeFirstFinalize); expect(firstFinalizeCaptures[0]?.[0]).toMatchObject({ runId: fixture.runId, runStatus: "failed", }); await db.update(issues).set({ status: issueStatus }).where(eq(issues.id, fixture.issueId)); const publish = vi.fn(); await deliverExecutionStatuses(db, { publish }); expect(publish.mock.calls.filter(([event]) => event.payload.runId === fixture.runId)).toHaveLength(1); const [beforeReplay] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); expect(beforeReplay.executionStatusDeliveryId).toBeNull(); publish.mockClear(); const callsBefore = mockTelemetryClient.track.mock.calls.length; const captureCallsBeforeReplay = mockCaptureRunFailure.mock.calls.length; // Simulate several periodic sweeps, including overlapping reconciliation. for (let sweep = 0; sweep < 3; sweep += 1) { await Promise.all([0, 1].map(() => finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, }))); await deliverExecutionStatuses(db, { publish }); } expect(publish.mock.calls.filter(([event]) => event.payload.runId === fixture.runId)).toHaveLength(0); const [afterReplay] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); expect(afterReplay).toMatchObject({ executionStatusDeliveryId: null, updatedAt: beforeReplay.updatedAt, nativePhaseUpdatedAt: beforeReplay.nativePhaseUpdatedAt, finishedAt: beforeReplay.finishedAt, }); expect(agentTaskRunCalls(callsBefore)).toHaveLength(0); expect(captureRunFailureCallsFrom(captureCallsBeforeReplay)).toHaveLength(0); }); it("repairs an incomplete committed failure once and preserves its pending delivery on replay", async () => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture, { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, turnTerminalState: "failed", runTerminalState: "failed", }); const finalize = () => finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); await finalize(); await db.update(heartbeatRuns).set({ finishedAt: null, nativePhase: "arbitrating", executionStatusDeliveryId: null, }).where(eq(heartbeatRuns.id, fixture.runId)); await finalize(); const [repaired] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); expect(repaired.status).toBe("failed"); expect(repaired.nativePhase).toBe("committed"); expect(repaired.finishedAt).toBeInstanceOf(Date); expect(repaired.executionStatusDeliveryId).toBeTypeOf("string"); await finalize(); const [replayed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); expect(replayed).toEqual(repaired); }); it("reports a committed failure once when the run row was an unreported pending retry", async () => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture, { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, turnTerminalState: "failed", runTerminalState: "failed", }); const finalize = () => finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); await finalize(); // The executor stores a scheduled same-run retry as failed + retryable_failure, // and reportRunFailure skips that write. await db.update(heartbeatRuns).set({ nativePhase: "retryable_failure", }).where(eq(heartbeatRuns.id, fixture.runId)); await waitForPendingRunFailureReports(); const captureCallsBefore = mockCaptureRunFailure.mock.calls.length; await finalize(); await waitForPendingRunFailureReports(); expect(captureRunFailureCallsFrom(captureCallsBefore)).toEqual([ [expect.objectContaining({ runId: fixture.runId, runStatus: "failed" })], ]); await finalize(); await waitForPendingRunFailureReports(); expect(captureRunFailureCallsFrom(captureCallsBefore)).toHaveLength(1); }); it("retires this run's scheduled finalization retry after successful commit", async () => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture, CONTROL_PLANE_CONFORMANCE_TERMINAL); await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error("native_workspace_sync_out_failed"), failureScope: "workspace", projectRunStatus: true }); const [pending] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId)); expect(pending).toMatchObject({ status: "active", evidence: { runId: fixture.runId } }); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const [settled] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, pending.id)); expect(settled.status).toBe("resolved"); }); it("repairs stale retry metadata on an already committed successful run", async () => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture, CONTROL_PLANE_CONFORMANCE_TERMINAL); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); await db.update(heartbeatRuns).set({ resultJson: { ...before.resultJson, finalizationPhase: "retryable_failure", failureCode: "native_finalization_invalid", originalFailureCode: "native_finalization_invalid", nextAttemptAt: new Date().toISOString(), } }).where(eq(heartbeatRuns.id, fixture.runId)); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); expect(after).toMatchObject({ status: "succeeded", nativePhase: "committed", resultJson: { finalizationPhase: "committed", failureCode: null, originalFailureCode: null, nextAttemptAt: null, } }); }); it("ignores a stale workspace failure after the same run committed successfully", async () => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture, CONTROL_PLANE_CONFORMANCE_TERMINAL); await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); const late = await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error("native_workspace_sync_out_failed"), failureScope: "workspace", projectRunStatus: true, }); expect(late.phase).toBe("committed"); const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); expect(after).toEqual(before); expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))).toEqual([]); }); it("materializes recovery for an agent-owned invalid-result outcome", async () => { const fixture = await seedNativeRun(); await db.insert(nativeRunFinalizations).values({ runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "terminal_failure", failureCode: "native_finalization_invalid", failureDetail: { recoveryOwner: { kind: "agent", agentId } }, }); const outcome = await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error("native_finalization_invalid"), }); expect(outcome.phase).toBe("retryable_failure"); expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))) .toEqual([expect.objectContaining({ status: "active", ownerType: "agent", cause: "native_finalization_invalid" })]); }); it.each(["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_unsafe_archive"])( "does not reopen board-owned %s repair for a late failure", async (failureCode) => { const fixture = await seedNativeRun(); await db.insert(nativeRunFinalizations).values({ runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "workspace_finalizing", }); await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error(failureCode), failureScope: "workspace", permanent: true }); const [before] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)); const recoveryBefore = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId)); for (const failureScope of [undefined, "workspace"] as const) { await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error("native_workspace_sync_out_failed"), failureScope }); } const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)); expect(after).toEqual(before); expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))).toEqual(recoveryBefore); }); it("emits zero events when a retryable-failure write's conditional status spread is omitted", async () => { const fixture = await seedNativeRun(); // recordNativeFinalizationFailure only needs the run and coordinator rows // to exist — no persisted native result is required. await db.insert(nativeRunFinalizations).values({ runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "observed", attempt: 0, }); const callsBefore = mockTelemetryClient.track.mock.calls.length; // projectRunStatus is omitted (falsy): the conditional spread in // recordRetryableFailure never includes `status`, so the write never sets // a terminal status, regardless of the attempt count. await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error("native_test_failure"), }); expect(agentTaskRunCalls(callsBefore)).toHaveLength(0); const run = await db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)) .then((rows) => rows[0]); expect(run?.status).toBe("running"); }); it("bounds workspace-only retries without consuming the provider attempt", async () => { const fixture = await seedNativeRun(); await db.insert(nativeRunFinalizations).values({ runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "observed", attempt: 1, }); for ( let expectedAttempt = 1; expectedAttempt <= 3; expectedAttempt += 1 ) { await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error("native_workspace_sync_out_failed"), projectRunStatus: true, failureScope: "workspace", }); const coordinator = await db .select() .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, fixture.runId)) .then((rows) => rows[0]!); expect(coordinator.attempt).toBe(1); expect(coordinator.failureDetail).toMatchObject({ workspaceFinalizeAttempt: expectedAttempt, }); expect(coordinator.phase).toBe( expectedAttempt === 3 ? "terminal_failure" : "retryable_failure", ); } await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)) .then((rows) => rows[0]?.status), ).resolves.toBe("failed"); }); it("blocks immediately when the sandbox with unexported changes is gone", async () => { const fixture = await seedNativeRun(); await db.insert(nativeRunFinalizations).values({ runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "workspace_finalizing", attempt: 1, resultId: null, }); const failure = await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error("native_workspace_sync_out_unrecoverable"), projectRunStatus: true, failureScope: "workspace", permanent: true, }); expect(failure).toMatchObject({ phase: "terminal_failure", failureCode: "native_workspace_sync_out_unrecoverable", nextAttemptAt: null, attempt: 1, }); await expect( db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)) .then((rows) => rows[0]?.status), ).resolves.toBe("failed"); await expect( db .select({ status: issues.status }) .from(issues) .where(eq(issues.id, fixture.issueId)) .then((rows) => rows[0]?.status), ).resolves.toBe("blocked"); }); it("commits the saved result when unsafe workspace export is omitted, with only a run log", async () => { const fixture = await seedNativeRun(); await db.update(completionContracts).set({ risk: "low", completionAuthority: "agent_claim_policy" }) .where(eq(completionContracts.id, fixture.contractId)); await driveToCompleteResult(fixture, CONTROL_PLANE_CONFORMANCE_TERMINAL, { ...CONTROL_PLANE_CONFORMANCE_RESULT, completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim!, contractRevision: "telemetry-v1" }, }); const [original] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, fixture.runId)); const restore = vi.fn(async () => { throw new Error("Daytona syncOut refusing tarball link whose target escapes the extraction dir: tools/pnpm -> /private/tool"); }); const assertOwnership = vi.fn(async () => {}); await restoreNativeWorkspaceBestEffort({ db, runId: fixture.runId, restore, assertOwnership }); const outcome = await finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, preserveProviderAttempt: true }); expect(outcome.phase).toBe("committed"); expect(restore).toHaveBeenCalledOnce(); expect(assertOwnership).toHaveBeenCalledOnce(); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); const [issue] = await db.select().from(issues).where(eq(issues.id, fixture.issueId)); expect(run).toMatchObject({ status: "succeeded", error: null, errorCode: null }); expect(issue.status).toBe("done"); const results = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, fixture.runId)); expect(results).toEqual([original]); const actions = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId)); expect(actions).toEqual([]); const events = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, fixture.runId)); expect(events.filter(event => event.eventType === "workspace_export_omitted")).toMatchObject([ { level: "info", payload: { reason: "restore_unsafe_archive" } }, ]); expect(JSON.stringify(events)).not.toContain("/private/tool"); }); it.each(["active", "resolved", "missing"])("automatically completes a legacy unsafe result with a %s repair action and no sandbox", async actionState => { const fixture = await seedNativeRun(); await db.update(completionContracts).set({ risk: "low", completionAuthority: "agent_claim_policy" }).where(eq(completionContracts.id, fixture.contractId)); await driveToCompleteResult(fixture, CONTROL_PLANE_CONFORMANCE_TERMINAL, { ...CONTROL_PLANE_CONFORMANCE_RESULT, completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim!, contractRevision: "telemetry-v1" }, }); const originalResults = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, fixture.runId)); await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_workspace_sync_out_unsafe_archive" }).where(eq(nativeRunFinalizations.runId, fixture.runId)); await db.update(heartbeatRuns).set({ status: "failed", nativePhase: "terminal_failure", errorCode: "native_workspace_sync_out_unsafe_archive", runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v1", state: "prepared", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: "missing-old-sandbox", remoteCwd: "/work", resourceDisposition: "destroy" } }, }).where(eq(heartbeatRuns.id, fixture.runId)); await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, fixture.issueId)); if (actionState !== "missing") await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: fixture.issueId, kind: "active_run_watchdog", ownerType: "board", cause: "native_workspace_sync_out_unsafe_archive", fingerprint: fixture.runId, evidence: { runId: fixture.runId }, nextAction: "Repair the unsafe link manually", status: actionState, ...(actionState === "resolved" ? { outcome: "restored", resolutionNote: "new_source_execution_path", resolvedAt: new Date() } : {}), }); const priorWakeups = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); const priorActivity = await db.select().from(activityLog).where(eq(activityLog.entityId, fixture.issueId)); await recoverLegacyUnsafeWorkspaceExports(db, [fixture.runId]); expect(await db.select().from(activityLog).where(eq(activityLog.entityId, fixture.issueId))).toEqual(priorActivity); // No environment runtime is supplied: a stopped, deleted, or unavailable // provider cannot prevent omission and accepted-result commitment. await reconcileNativeFinalizations(db, [fixture.runId]); await reconcileNativeFinalizations(db, [fixture.runId]); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)))[0]).toMatchObject({ status: "succeeded", nativePhase: "committed", error: null, errorCode: null }); expect((await db.select().from(issues).where(eq(issues.id, fixture.issueId)))[0].status).toBe("done"); expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, fixture.runId))).toEqual(originalResults); expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).toEqual(priorWakeups); expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.nativeIssueId, fixture.issueId))).toHaveLength(1); const actions = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId)); expect(actions.every(action => action.status === "resolved" && action.nextAction === "")).toBe(true); expect((await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, fixture.runId)))[0]).toMatchObject({ status: "succeeded", metadata: { workspaceSync: { omitted: true, legacy: true } } }); const events = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, fixture.runId)); expect(events.filter(event => event.eventType === "workspace_export_omitted")).toMatchObject([{ level: "info", payload: { reason: "restore_unsafe_archive", legacy: true } }]); expect(events.filter(event => event.eventType === "workspace_export_omitted")).toHaveLength(1); }); it("emits exactly one event for a cancel_continuations write (trap 2: :685/:518 overlap)", async () => { const fixture = await seedNativeRun(); // Build the minimal real rows commitNativeStatusDecision's foreign keys // require, without going through the evidence-classifier pipeline. const resultId = randomUUID(); await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId: fixture.issueId, runId: fixture.runId, completionContractId: fixture.contractId, serverFingerprint: `fp-${resultId}`, schemaStatus: "accepted", resultJson: {}, canonicalSha256: `sha-${resultId}`, }); const assessmentId = randomUUID(); await db.insert(workAssessments).values({ id: assessmentId, companyId, issueId: fixture.issueId, runId: fixture.runId, contractId: fixture.contractId, resultId, triggerKind: "native_result", triggerActorCompanyId: companyId, priorIssueStatus: "in_progress", priorStatusVersion: 0, policyVersion: "telemetry-v1", assessmentJson: {}, inputDigest: `digest-${assessmentId}`, }); await db.insert(nativeRunFinalizations).values({ runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "arbitrating", attempt: 0, resultId, }); // Shaped exactly like resolveNativeCancellationStatus("issue", ...) in // native-session-executor.ts — the only place cancel_continuations is // produced today. const decision: NativeStatusDecision = { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "cancelled", toStatus: "cancelled", reasonCode: "cancellation_issue_authorized", unblockDescriptor: null, effects: [ { kind: "release_checkout" }, { kind: "cancel_continuations" }, ], }; const callsBefore = mockTelemetryClient.track.mock.calls.length; await commitNativeStatusDecision({ db, companyId, issueId: fixture.issueId, runId: fixture.runId, assessmentId, priorStatus: "in_progress", priorStatusVersion: 0, priorDecisionId: null, decision, }); const afterCommit = agentTaskRunCalls(callsBefore); // status-decision-committer.ts:685 (the cancel_continuations effect) // wrote this run's terminal status and emitted for it. expect(afterCommit).toHaveLength(1); expect(afterCommit[0]?.[1]).toMatchObject({ agent_id: agentId, state: "cancelled" }); const run = await db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, fixture.runId)) .then((rows) => rows[0]); expect(run?.status).toBe("cancelled"); // native-run-finalizer.ts:518's own guard: it skips its emit whenever the // decision it just committed already included cancel_continuations, // because :685 above already wrote and emitted for this run. Reproduce // the exact guard predicate on the same decision object to prove it // resolves to "skip", so the run's total event count for this commit // stays at exactly one even after the finalizer's own write runs. const alreadyEmittedByCommittedDecision = decision.effects.some( (effect) => effect.kind === "cancel_continuations", ); expect(alreadyEmittedByCommittedDecision).toBe(true); expect(agentTaskRunCalls(callsBefore)).toHaveLength(1); }); it.each([true, false])("signals instruction collection only when native failure projects terminal status (%s)", async projectRunStatus => { const fixture = await seedNativeRun(); await driveToCompleteResult(fixture); const settled = vi.fn(); const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.instructionCollection, settled); try { await recordNativeFinalizationFailure({ db, runId: fixture.runId, error: new Error("native_fixture_permanent_failure"), permanent: true, projectRunStatus }); expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)))[0].phase).toBe("terminal_failure"); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); if (projectRunStatus) { expect(run.status).not.toBe("running"); expect(settled).toHaveBeenCalled(); } else { expect(run.status).toBe("running"); expect(settled).not.toHaveBeenCalled(); } } finally { unsubscribe(); } }); it("signals durable accounting when native commitment or permanent failure makes a receipt actionable", async () => { const settled = vi.fn(); const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.accounting, settled); try { const committed = await seedNativeRun(); await driveToCompleteResult(committed); const beforeCommit = settled.mock.calls.length; await finalizeNativeRun({ db, runId: committed.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, committed.runId)))[0].phase).toBe("committed"); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, committed.runId)))[0].status).toBe("succeeded"); expect(settled.mock.calls.length).toBeGreaterThan(beforeCommit); const failed = await seedNativeRun(); await driveToCompleteResult(failed); const beforeFailure = settled.mock.calls.length; await recordNativeFinalizationFailure({ db, runId: failed.runId, error: new Error("native_fixture_permanent_failure"), permanent: true }); expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, failed.runId)))[0].phase).toBe("terminal_failure"); expect(settled.mock.calls.length).toBeGreaterThan(beforeFailure); } finally { unsubscribe(); } }); }); enableRunFailureReporting();