diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index 70cb7cd8ec9..df854e62152 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, agentWakeupRequests, companies, createDb, applyPendingMigrations, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { agents, agentWakeupRequests, companies, createDb, applyPendingMigrations, heartbeatRuns, issues, nativeRunFinalizations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { taskInterruptionPresentation, taskInterruptionPresentations } from "../services/task-interruption-presentation.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -24,6 +24,28 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + it("shows a pending native same-run resume instead of a competing Retry", async () => { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Resuming task", issuePrefix: `R${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Resuming agent", adapterType: "paperclip_runner" }); + const [issue] = await db.insert(issues).values({ companyId, title: "Resume after attach failure", + status: "in_progress", assigneeAgentId: agentId }).returning(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issue.id, scopeKind: "issue", + status: "failed", runtimeMode: "native", errorCode: "native_session_interrupted", finishedAt: new Date() }); + const resumeAt = new Date(Date.now() + 30_000); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId, phase: "retryable_failure", + attempt: 1, nextAttemptAt: resumeAt }); + for (const presentation of [(await taskInterruptionPresentations(db, companyId, [issue])).get(issue.id), + await taskInterruptionPresentation(db, issue)]) { + expect(presentation).toMatchObject({ state: "retry_scheduled", canRetry: false, nextRetryAt: resumeAt.toISOString() }); + } + // Exhausted recovery is terminal: exactly one Retry returns. + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", nextAttemptAt: null }) + .where(eq(nativeRunFinalizations.runId, runId)); + expect((await taskInterruptionPresentations(db, companyId, [issue])).get(issue.id)) + .toMatchObject({ state: "failed", canRetry: true }); + }); + it("redacts workspace preparation errors on task detail and list reads", async () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); try { diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index adbc79479a5..fab317dc824 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -80,6 +80,24 @@ describe("unadmitted chat Retry presentation", () => { }); }); +// The native coordinator resumes the same run itself after an attach failure. +// A manual Retry offered meanwhile would race that automatic resume. +it("shows a pending native same-run resume instead of a competing Retry", () => { + const now = Date.parse("2026-11-06T15:25:50.000Z"); + const failed = { ...run, runtimeMode: "native" as const, errorCode: "native_session_interrupted", resultJson: null }; + const resumeAt = new Date("2026-11-06T15:26:18.687Z"); + expect(projectTaskInterruption(issue, { run: { ...failed, nativeResumeAt: resumeAt }, agent, paused: false, approval: false }, now)) + .toMatchObject({ state: "retry_scheduled", canRetry: false, nextRetryAt: resumeAt.toISOString() }); + // A due resume is being dispatched; it is still not the user's turn. + expect(projectTaskInterruption(issue, { run: { ...failed, nativeResumeAt: new Date(now - 30_000) }, agent, paused: false, approval: false }, now)) + .toMatchObject({ state: "working", canRetry: false }); + // A resume that never ran, or no resume at all, leaves exactly one Retry. + expect(projectTaskInterruption(issue, { run: { ...failed, nativeResumeAt: new Date(now - 10 * 60_000) }, agent, paused: false, approval: false }, now)) + .toMatchObject({ state: "failed", canRetry: true }); + expect(projectTaskInterruption(issue, { run: failed, agent, paused: false, approval: false }, now)) + .toMatchObject({ state: "failed", canRetry: true }); +}); + it("explains a persisted Retry wait without exposing the saved payload", () => { const failed = { ...run, errorCode: "native_provider_terminal_failed", resultJson: null }; const wait = { id: "retry", requestedAt: new Date(), sourceRunId: failed.id, diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 59797d3958e..dec56770f76 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -1,6 +1,6 @@ import { and, desc, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, agentWakeupRequests, approvals, chatActions, companies, heartbeatRuns, issueApprovals, issueRecoveryActions, - issues, issueTreeHoldMembers, issueTreeHolds, projects, type Db } from "@paperclipai/db"; + issues, issueTreeHoldMembers, issueTreeHolds, nativeRunFinalizations, projects, type Db } from "@paperclipai/db"; import { unionAll } from "drizzle-orm/pg-core"; import { HEARTBEAT_RUN_STATUSES, readProviderFailure, type TaskInterruptionPresentation } from "@paperclipai/shared"; import { captureTaskInterruptionFailure } from "../sentry.js"; @@ -26,11 +26,17 @@ import { AI_CONNECTION_BUSY_RETRY_REASON, AI_CONNECTION_POOL_WAIT_RETRY_REASON, const resourceWaitRetryReasons = new Set([WORKSPACE_BUSY_RETRY_REASON, AI_CONNECTION_BUSY_RETRY_REASON, AI_CONNECTION_POOL_WAIT_RETRY_REASON, RUNNER_REVISION_WAIT_RETRY_REASON]); +// A due same-run resume dispatches within seconds; past this, offer Retry. +const NATIVE_RESUME_DISPATCH_GRACE_MS = 2 * 60_000; + type ProjectionIssue = Pick; type ProjectionRun = Pick & Partial>; + | "scheduledRetryReason">> & { + /** Due time of the native coordinator's own same-run resume, if one is pending. */ + nativeResumeAt?: Date | null; +}; type ProjectionWait = { id: string; requestedAt: Date; sourceRunId?: string | null; executionWaitReason?: string | null; preparationStatus?: string | null; preparationRetryAt?: number | string | null }; type ProjectionState = { @@ -148,10 +154,16 @@ export function projectTaskInterruption(issue: ProjectionIssue, input: Projectio } if (run.status === "scheduled_retry" && resourceWaitRetryReasons.has(run.scheduledRetryReason ?? "") && !system) return { ...base, state: "working" }; - const retryAt = system?.nextAttemptAt ?? run.scheduledRetryAt?.getTime(); + // The native coordinator resumes this same run by itself. A manual Retry + // would race it, so show the resume until it is due and briefly while its + // dispatch runs. A resume that never ran falls through to one Retry. + const nativeResumeAt = run.nativeResumeAt?.getTime(); + const retryAt = system?.nextAttemptAt ?? nativeResumeAt ?? run.scheduledRetryAt?.getTime(); if (retryAt && retryAt > now) return { ...base, state: "retry_scheduled", - nextRetryAt: new Date(retryAt).toISOString(), canRetry: !preparationPending, + nextRetryAt: new Date(retryAt).toISOString(), canRetry: !preparationPending && !nativeResumeAt, ...(system?.nextAttemptAt && recoveryStep ? { recoveryStep } : {}) }; + if (nativeResumeAt && !system && now - nativeResumeAt < NATIVE_RESUME_DISPATCH_GRACE_MS) + return { ...base, state: "working" }; if (recoveryRunning) return { ...base, state: "working", ...(recoveryStep ? { recoveryStep } : {}) }; if (retryWait?.sourceRunId === run.id && retryWait.executionWaitReason) { const reason = Object.hasOwn(retryWaitMessages, retryWait.executionWaitReason) ? retryWait.executionWaitReason : "execution_recovery"; @@ -203,6 +215,10 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i or ${heartbeatRuns.errorCode} = 'workspace_validation_failed' then ${heartbeatRuns.error} else null end`, errorCode: heartbeatRuns.errorCode, livenessState: heartbeatRuns.livenessState, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + nativeResumeAt: sql`(select ${nativeRunFinalizations.nextAttemptAt} from ${nativeRunFinalizations} + where ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId} and ${nativeRunFinalizations.runId} = ${heartbeatRuns.id} + and ${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} = 'failed' + and ${nativeRunFinalizations.resultId} is null and ${nativeRunFinalizations.phase} = 'retryable_failure')`.mapWith(value => value == null ? null : new Date(value)), // The run whose resultJson supplied providerFailure. A scheduled retry // shows its predecessor's reason, so redaction must use that run's registry. providerFailureRunId: sql`case when ${heartbeatRuns.resultJson}->'providerFailure' is null