import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, agentWakeupRequests, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issueThreadInteractions, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db"; import type { ControlPlanePort, NativeExecutionInputV1, PrpEvent, PrpTerminalState } from "@paperclipai/paperclip-runner"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/testing.js"; const provider = vi.hoisted(() => ({ execute: vi.fn() })); vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ ...(await importOriginal()), executeNativeSession: provider.execute, })); import { executePaperclipNativeSession } from "./native-session-executor.js"; import { buildNativeCompletionContract } from "./completion-contracts.js"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; import { NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js"; import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js"; import { finalizeNativeRun } from "./native-run-finalizer.js"; import { readPersistedNativeFailureEvidence, readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js"; import { heartbeatService } from "../heartbeat.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js"; import { claimQueuedNativeReviewRun } from "./native-review-dispatch.js"; import { commitNativeStatusDecision, NativeStatusRaceError } from "./status-decision-committer.js"; import { NATIVE_STATUS_ARBITER_POLICY_VERSION } from "./status-arbiter.js"; import { issueThreadInteractionService } from "../issue-thread-interactions.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js"; import { persistHeartbeatRunProcessMetadata } from "../heartbeat/run-lifecycle.js"; import { readNativeLocalProcessStop, releaseSettledNativeRunProcess } from "../native-local-process-stop.js"; const EXITED_PID = 4_194_304; describe("native completed failure diagnostics", () => { let temporary: Awaited>; let db: ReturnType; const companyId = randomUUID(); const agentId = randomUUID(); const workerId = randomUUID(); const model = "test-model"; beforeAll(async () => { temporary = await startEmbeddedPostgresTestDatabase("native-model-failure-"); db = createDb(temporary.connectionString); await db.insert(companies).values({ id: companyId, name: "Native failure regression", issuePrefix: "NMF" }); await db.insert(agents).values({ id: agentId, companyId, name: "Native failure agent", status: "active", adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model } }); await db.insert(agents).values({ id: workerId, companyId, name: "Review source worker", status: "active", adapterType: "paperclip_runner" }); }); afterAll(async () => { await temporary?.cleanup(); }); it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "opencode-lost-server", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)( "preserves accepted results and only labels the matching provider rejection (%s)", async (scenario) => { const issueId = randomUUID(); const runId = randomUUID(); const opencode = scenario === "opencode-lost-server"; const executionProvider = opencode ? { kind: "opencode" as const, model: `openrouter/${model}` } : { kind: "codex" as const, model }; const [issue] = await db.insert(issues).values({ id: issueId, companyId, title: "Explain a provider rejection", status: "in_progress", workMode: "standard", assigneeAgentId: agentId }).returning(); const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId } }).returning(); await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); const completionInput = buildNativeCompletionContract(issue, { revision: contract.revision }); await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); const execution: NativeExecutionInputV1 = { schema: "paperclip.native-execution-input.v1", binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, provider: executionProvider, task: { identifier: issueId, title: issue.title, description: null, prompt: "Reply to the current message.", workMode: "standard" }, workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, session: { normalizedSessionId: native.normalizedSessionId, driverKind: opencode ? "opencode_server" : "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completionInput }, interactionResponses: [], credentialBindings: [], }; await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); if (scenario.endsWith("-review")) { const sourceRunId = randomUUID(), sourceResultId = randomUUID(), assessmentId = randomUUID(); const decisionId = randomUUID(), interactionId = randomUUID(); await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId: workerId, nativeIssueId: issueId, runtimeMode: "native", status: "succeeded", completionContractId: contract.id, completionContractSha256: contract.canonicalSha256 }); await db.insert(nativeRunResults).values({ id: sourceResultId, companyId, issueId, runId: sourceRunId, completionContractId: contract.id, serverFingerprint: sourceResultId, schemaStatus: "accepted", resultJson: {}, canonicalSha256: sourceResultId }); await db.insert(workAssessments).values({ id: assessmentId, companyId, issueId, runId: sourceRunId, contractId: contract.id, resultId: sourceResultId, triggerKind: "native_result", triggerActorCompanyId: companyId, priorIssueStatus: "in_progress", priorStatusVersion: 0, policyVersion: "fixture", assessmentJson: {}, inputDigest: assessmentId }); const [reviewIssue] = await db.update(issues).set({ status: "in_review", assigneeAgentId: workerId }).where(eq(issues.id, issueId)).returning(); await db.insert(statusDecisions).values({ id: decisionId, companyId, issueId, runId: sourceRunId, assessmentId, decisionVersion: 1, policyVersion: "fixture", fromStatus: "in_progress", toStatus: "in_review", reasonCode: "explicit_review", decisionJson: { projectedStatusVersion: reviewIssue.statusVersion }, decisionDigest: decisionId, applicationState: "applied" }); await db.update(issues).set({ lastStatusDecisionId: decisionId }).where(eq(issues.id, issueId)); await db.insert(issueThreadInteractions).values({ id: interactionId, companyId, issueId, kind: "request_confirmation", sourceRunId, addresseeAgentId: agentId, createdByAgentId: workerId, continuationPolicy: "wake_assignee", status: scenario === "resolved-review" ? "accepted" : "pending", ...(scenario === "resolved-review" ? { resolvedByRunId: runId, resolvedByAgentId: agentId } : {}), payload: { version: 1, prompt: "Review the completed work.", target: { type: "custom", key: "native_completion_review", revisionId: decisionId } }, }); const contextSnapshot = { issueId, wakeReason: "native_completion_review", nativeReviewInteractionId: interactionId, nativeReviewDecisionId: decisionId }; await db.update(heartbeatRuns).set({ contextSnapshot }).where(eq(heartbeatRuns.id, runId)); if (scenario === "superseded-review") await db.update(issues).set({ lastStatusDecisionId: null }).where(eq(issues.id, issueId)); const review = await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot, allowResolvedByRunId: runId }); if (scenario === "superseded-review") expect(review).toBeNull(); else expect(review?.interaction.status).toBe(scenario === "resolved-review" ? "accepted" : "pending"); } const event: PrpEvent = { schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId, turnId: native.turnId, sourceInstanceId: native.runnerInstanceId, sourceEventId: "model-terminal", sourceSeq: 1, sourceKind: "runner", eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), payload: opencode ? { status: "failed", providerTerminalObserved: false, error: { code: "opencode_server_exited", message: "Private stream failure", opencodeFailure: { kind: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" }, } } : { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({ type: "error", status: 400, error: { type: "invalid_request_error", message: scenario === "unknown-error" ? "Private unknown failure" : `The '${model}' model is not supported when using Codex with a ChatGPT account.` }, }) } }, }; const openInput = { identity: { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId }, backendKind: "mock" as const, sourceInstanceId: native.runnerInstanceId }; if (scenario === "duplicate") { const prior = new PaperclipControlPlanePort(db, { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId, sourceInstanceId: native.runnerInstanceId, controlPlaneSourceInstanceId: "prior-controller", completionContractId: contract.id, completionContractSha256: contract.canonicalSha256 }); await prior.openRun(openInput); expect((await prior.appendEvent(event)).disposition).toBe("committed"); } const turnId = scenario === "different-turn" ? "later-turn" : native.turnId; const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: scenario === "succeeded" ? "completed" : scenario === "cancelled" ? "cancelled" : "failed", runTerminalState: scenario === "succeeded" ? "succeeded" : scenario === "cancelled" ? "cancelled" : "failed", reportedWorkDisposition: "blocked", }; const result = { ...CONTROL_PLANE_CONFORMANCE_RESULT, reportedWorkDisposition: "blocked" as const, blocker: { reasonCode: "provider_rejected", owner: { kind: "user" as const, name: "Task owner" }, unblockAction: "Select a compatible connection.", scope: "task_wide" as const }, completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim, contractRevision: completionInput.revision } }; provider.execute.mockReset().mockImplementation(async (options: { controlPlane: ControlPlanePort }) => { await options.controlPlane.openRun(openInput); if (scenario === "untrusted-source") { await expect(options.controlPlane.appendEvent({ ...event, sourceInstanceId: "wrong-runner" })).rejects.toThrow("native_event_source_binding_mismatch"); } else if (scenario === "untrusted-run" || scenario === "untrusted-session") { await expect(options.controlPlane.appendEvent({ ...event, ...(scenario === "untrusted-run" ? { runId: randomUUID() } : { normalizedSessionId: "wrong-session" }), })).rejects.toThrow("native_event_binding_mismatch"); } else { expect((await options.controlPlane.appendEvent(event)).disposition).toBe(scenario === "duplicate" ? "duplicate" : "committed"); } await options.controlPlane.completeRun({ result, terminal, turnId }); return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: execution.session.driverKind, driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; }); const adapter = await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { descriptor: async () => { throw new Error("Unexpected provider call"); }, openSession: async () => { throw new Error("Unexpected provider call"); }, } }); const classified = scenario === "committed" || scenario === "duplicate" || scenario.endsWith("-review") || scenario === "reassigned" || scenario === "successor-race"; const reviewAuthorityEnded = scenario === "superseded-review" || scenario === "resolved-review"; const ownerChanged = scenario === "reassigned" || scenario === "successor-race"; expect(adapter.errorCode).toBe(classified ? "native_provider_model_rejected" : undefined); expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`); expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : undefined); // Only a committed, unclassified failure of the terminal turn explains the generic message. const turnFailure = scenario === "unknown-error" ? { provider: "codex", category: "turn_failed", codexErrorInfo: "other" } : opencode ? { provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_server_exited", streamFailure: "server_exited", signal: "SIGKILL", connectionError: "ECONNREFUSED" } : undefined; expect(adapter.resultJson?.nativeTurnFailure).toEqual(turnFailure); expect(adapter.nativeFinalization).toMatchObject({ result, terminal, turnId, workspaceFinalizeStatus: "pending" }); expect(provider.execute).toHaveBeenCalledTimes(1); expect(execution.provider).toEqual(executionProvider); const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); expect(coordinator).toMatchObject({ phase: "workspace_finalizing", leaseOwner: null, attempt: 1, nextAttemptAt: null }); expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toHaveLength(1); expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0); expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toHaveLength(scenario.startsWith("untrusted-") ? 0 : 1); const diagnostics = collectRunFailureDiagnostics({ ...run, resultJson: adapter.resultJson ?? null }, {}); expect(diagnostics.provider).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : turnFailure ?? {}); expect(JSON.stringify(diagnostics)).not.toContain(model); expect(JSON.stringify(diagnostics)).not.toContain("Private"); const [persistedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); expect(persistedRun.errorCode).toBeNull(); // No adapter metadata has been saved yet. expect(await readPersistedNativeModelRejection(db, persistedRun, turnId, terminal)).toEqual(classified ? { errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: NATIVE_MODEL_REJECTION_DIAGNOSTIC, } : null); expect((await readPersistedNativeFailureEvidence(db, persistedRun, turnId, terminal)).turnFailure).toEqual(turnFailure ?? null); if (classified) { if (scenario === "committed") { // Neither saved display diagnostics nor a different execution binding // can confer the right to change the finalizer's retry decision. for (const changed of [ { ...persistedRun, runnerProfileJson: null }, { ...persistedRun, nativeSessionId: "foreign-session" }, { ...persistedRun, runnerInstanceId: "foreign-source" }, { ...persistedRun, companyId: randomUUID() }, { ...persistedRun, completionContractSha256: "foreign-contract" }, { ...persistedRun, runnerProfileJson: { nativeExecutionInput: { ...execution, provider: { kind: "codex", model: "different-model" } } } }, { ...persistedRun, runnerProfileJson: { nativeExecutionInput: { ...execution, binding: { ...execution.binding, runId: randomUUID() } } } }, ]) expect(await readPersistedNativeModelRejection(db, changed, turnId, terminal)).toBeNull(); const [stored] = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId)); await db.update(heartbeatRunEvents).set({ sourcePayloadSha256: "invalid-hash" }).where(eq(heartbeatRunEvents.id, stored.id)); expect(await readPersistedNativeModelRejection(db, persistedRun, turnId, terminal)).toBeNull(); await db.update(heartbeatRunEvents).set({ sourcePayloadSha256: stored.sourcePayloadSha256 }).where(eq(heartbeatRunEvents.id, stored.id)); } // Simulate a restart after native result acceptance, before heartbeat's // adapter update. Finalization must use durable provider evidence. const acceptedBefore = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId)); if (scenario === "reassigned") { await db.update(issues).set({ assigneeAgentId: workerId, executionRunId: null }).where(eq(issues.id, issueId)); } else if (scenario === "successor-race") { const [snapshot] = await db.select().from(issues).where(eq(issues.id, issueId)); const newerRunId = randomUUID(), raceAssessmentId = randomUUID(); await db.insert(heartbeatRuns).values({ id: newerRunId, companyId, agentId, status: "running", nativeIssueId: issueId }); await db.update(issues).set({ executionRunId: newerRunId }).where(eq(issues.id, issueId)); const [changed] = await db.select().from(issues).where(eq(issues.id, issueId)); expect(changed.statusVersion).toBe(snapshot.statusVersion); await db.insert(workAssessments).values({ id: raceAssessmentId, companyId, issueId, runId, contractId: contract.id, resultId: acceptedBefore[0].id, triggerKind: "native_result", triggerActorCompanyId: companyId, priorIssueStatus: snapshot.status, priorStatusVersion: snapshot.statusVersion, policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, assessmentJson: {}, inputDigest: raceAssessmentId }); await expect(commitNativeStatusDecision({ db, companyId, issueId, runId, assessmentId: raceAssessmentId, priorStatus: snapshot.status, priorStatusVersion: snapshot.statusVersion, priorDecisionId: snapshot.lastStatusDecisionId, requireModelRejectionOwner: { agentId, reviewContext: null }, decision: { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "blocked", toStatus: "blocked", reasonCode: "native_provider_model_rejected", unblockDescriptor: { owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }, effects: [{ kind: "bind_blocker", owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }] }, })).rejects.toBeInstanceOf(NativeStatusRaceError); expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId))).toHaveLength(0); } for (let retry = 0; retry < 2; retry++) { await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); } expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toEqual(acceptedBefore); const [finalRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); expect(finalRun).toMatchObject({ status: "failed", errorCode: "native_provider_model_rejected", error: NATIVE_MODEL_REJECTION_MESSAGE }); expect(finalRun.resultJson?.nativeTurnFailure).toBeUndefined(); const [finalIssue] = await db.select().from(issues).where(eq(issues.id, issueId)); expect(finalIssue.status).toBe(scenario.endsWith("-review") ? "in_review" : ownerChanged ? "in_progress" : "blocked"); const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); expect(decisions).toHaveLength(1); expect(decisions[0].reasonCode).toBe(reviewAuthorityEnded ? "native_review_action_finished" : ownerChanged ? "run_failed_partial_evidence_preserved" : "native_provider_model_rejected"); const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(ownerChanged ? 1 : 0); if (!ownerChanged) expect(await heartbeatService(db).dispatchPendingNativeStatusWakeups({ companyId })).toEqual({ scanned: 0, dispatched: 0, recovered: 0, deferred: 0 }); expect(provider.execute).toHaveBeenCalledTimes(1); if (scenario === "pending-review") { const reviewContext = readNativeReviewAssignmentContext(finalRun.contextSnapshot)!; expect(finalIssue).toMatchObject({ lastStatusDecisionId: reviewContext.nativeReviewDecisionId, executionRunId: null, checkoutRunId: null }); const originalReview = await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot: reviewContext }); expect(originalReview?.interaction.status).toBe("pending"); expect(finalIssue.statusVersion).toBe(originalReview!.sourceDecision.decisionJson.projectedStatusVersion); expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0); // Exercise the real release and periodic recovery lanes, rather than // assuming that absence of a finalizer wake prevents later retries. const unexpected = vi.fn(async () => { throw new Error("Unexpected automatic recovery"); }); const release = createReleaseIssueExecution({ issueLock: createPostgresWakeQueueAdapter(db, { resolveResponsibleUserId: unexpected, getRoutineEnv: unexpected, resolveSessionBeforeForWakeup: unexpected }), recovery: { escalateStrandedAssignedIssue: unexpected, escalateStrandedRecoveryIssueInPlace: unexpected }, }); expect((await release({ companyId, runId, now: new Date() })).outcome.kind).toBe("released"); await heartbeatService(db).reconcileStrandedAssignedIssues(); expect(unexpected).not.toHaveBeenCalled(); expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0); expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).filter(wake => wake.payload?.issueId === issueId)).toHaveLength(0); const [afterRecovery] = await db.select().from(issues).where(eq(issues.id, issueId)); expect(afterRecovery).toMatchObject({ status: "in_review", statusVersion: finalIssue.statusVersion, lastStatusDecisionId: finalIssue.lastStatusDecisionId }); // The operator repairs configuration and explicitly retries the // assigned reviewer. No review rebinding or status reset is needed. await db.update(agents).set({ adapterConfig: { provider: "codex", model: "repaired-model" }, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, agentId)); // Occupy the execution slot to exercise real wake admission without // starting a provider. The normal atomic review claim follows below. const occupiedRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: occupiedRunId, companyId, agentId, status: "running", startedAt: new Date() }); const retryRun = await heartbeatService(db, { runtimeEnv: {} }).wakeup(agentId, { failedRunId: runId, source: "on_demand", triggerDetail: "manual", reason: "retry_failed_run", requestedByActorType: "user", requestedByActorId: "review-repair-operator", payload: { issueId }, contextSnapshot: { ...reviewContext }, }); expect(retryRun).toMatchObject({ status: "queued", retryOfRunId: runId, contextSnapshot: { issueId, ...reviewContext } }); const retryRunId = retryRun!.id; await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, occupiedRunId)); expect(await claimQueuedNativeReviewRun(db, { run: retryRun!, agentNameKey: "reviewer", claimedAt: new Date(), claimValues: { nativeIssueId: issueId, runtimeMode: "native" } })) .toMatchObject({ id: retryRunId, status: "running" }); expect(await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot: reviewContext, actingRunId: retryRunId, issueExecutionRunId: retryRunId })).not.toBeNull(); await issueThreadInteractionService(db).acceptInteraction( { id: issueId, companyId, projectId: null, goalId: null, status: "in_review" }, reviewContext.nativeReviewInteractionId, {}, { agentId, runId: retryRunId }, ); const [resolved] = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, reviewContext.nativeReviewInteractionId)); expect(resolved).toMatchObject({ status: "accepted", resolvedByAgentId: agentId, resolvedByRunId: retryRunId }); expect(await db.select().from(issues).where(eq(issues.id, issueId))).toMatchObject([{ status: "done", assigneeAgentId: workerId }]); expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toEqual(acceptedBefore); expect(provider.execute).toHaveBeenCalledTimes(1); await db.update(agents).set({ adapterConfig: { provider: "codex", model } }).where(eq(agents.id, agentId)); } } else if (scenario === "unknown-error" || opencode) { // A lookalike diagnostic flag does not alter unknown failed-result policy. await db.update(heartbeatRuns).set({ resultJson: { nativeProviderFailure: NATIVE_MODEL_REJECTION_DIAGNOSTIC } }).where(eq(heartbeatRuns.id, runId)); await persistHeartbeatRunProcessMetadata(db, runId, { pid: EXITED_PID, processGroupId: EXITED_PID, startedAt: new Date().toISOString(), targetKind: "local" }); await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" }); const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); expect(decision.reasonCode).toBe("run_failed_partial_evidence_preserved"); // Recovery finalized before heartbeat saved the adapter result, so the // turn failure must come from the persisted runner event. const [finalRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); expect(finalRun.resultJson?.nativeTurnFailure).toEqual(turnFailure); const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(1); if (scenario === "unknown-error") { await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const [settled] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); expect(coordinator.phase).toBe("committed"); expect(settled).toMatchObject({ status: "failed", processPid: EXITED_PID, processGroupId: EXITED_PID }); await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1_000, eventType: "harness.ready" }); expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); const [released] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); expect(released).toMatchObject({ status: "failed", processPid: null, processGroupId: null, processStartedAt: null }); expect(await readNativeLocalProcessStop(db, companyId, runId)).toMatchObject({ processPid: EXITED_PID, processGroupId: EXITED_PID }); } } }, ); });