diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 51fab29687c..6c1ccaca8ee 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -66112,7 +66112,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }, ); - it.each(["person", "person_with_input_before_stop", "person_after_held_admission", "person_paused_after_accept", "restart_recovery", "restart_recovery_joined_by_person"] as const)("continues an accepted GitHub task Retry from recovered saved work when its chat replay is refused after the click: %s", async initiator => { + it.each(["person", "person_with_input_before_stop", "person_after_held_admission", "person_paused_after_accept", "person_with_superseded_continuation", "restart_recovery", "restart_recovery_joined_by_person"] as const)("continues an accepted GitHub task Retry from recovered saved work when its chat replay is refused after the click: %s", async initiator => { const f = await currentGitHubTaskRetryFixture(initiator === "person_with_input_before_stop"); try { const companyId = f.fixture.companyId, issueId = f.issue.id; @@ -66215,6 +66215,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(joined).toMatchObject({ accepted: true, requestId: request.requestId, chatActionId: request.chatActionId }); } expect(request).toMatchObject({ accepted: true, stale: false, chatActionId: expect.any(String) }); + if (initiator === "person_with_superseded_continuation") await db.insert(heartbeatRuns).values({ companyId, + agentId: f.fixture.assignedAgentId, status: "cancelled", errorCode: "issue_execution_lock_changed", runtimeMode: "legacy", + issueId, retryOfRunId: sourceRunId, finishedAt: new Date(), contextSnapshot: { issueId } }); const chatAction = async () => (await db.select().from(chatActions).where(eq(chatActions.id, request.chatActionId!)))[0]; expect((await chatAction()).payload.sourceAuthorizationPending).toBe(true); diff --git a/server/src/__tests__/task-retry-banner-execution-owner.test.ts b/server/src/__tests__/task-retry-banner-execution-owner.test.ts new file mode 100644 index 00000000000..5010fe1c93b --- /dev/null +++ b/server/src/__tests__/task-retry-banner-execution-owner.test.ts @@ -0,0 +1,183 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { taskInterruptionPresentation, taskInterruptionPresentations } from "../services/task-interruption-presentation.js"; +import { requestTaskRetry, retireSupersededTaskRetry } from "../services/task-retry.js"; +import { heldTaskRetryRefusals } from "../services/task-retry-eligibility.js"; + +vi.mock("../sentry.js", () => ({ captureTaskInterruptionFailure: vi.fn() })); + +const support = await getEmbeddedPostgresTestSupport(); + +type RunSpec = Partial & { ageSeconds: number }; +type Row = { + name: string; + predecessor?: Partial; + followUp?: RunSpec; + continuation?: RunSpec; + lockOwner: "followUp" | "continuation" | null; + expected: { state: string; canRetry: boolean; observed: "predecessor" | "followUp" | "continuation"; retryAction?: "move_forward" }; + retryWith?: "predecessor" | "followUp" | "continuation"; + native?: boolean; + deliveredContinuation?: boolean; +}; + +const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000); +const lockChanged = { status: "cancelled", errorCode: "issue_execution_lock_changed", startedAt: null, + error: "Cancelled because another run owns task execution or checkout before replacement dispatch" } as const; +const continuationRun = { scheduledRetryReason: "native_agent_blocker_continuation" } as const; + +const STATE_TABLE: Row[] = [ + { name: "continuation scheduled while the follow-up turn runs and owns the lock", + followUp: { status: "running", startedAt: minutesAgo(0.5), ageSeconds: 100 }, + continuation: { ...continuationRun, status: "scheduled_retry", errorCode: "native_agent_blocker_continuation", + scheduledRetryAt: new Date(Date.now() + 30_000), ageSeconds: 30 }, + lockOwner: "followUp", expected: { state: "working", canRetry: false, observed: "followUp" } }, + { name: "continuation cancelled because the follow-up turn took the lock (t032 banner)", + followUp: { status: "running", startedAt: minutesAgo(0.5), ageSeconds: 100 }, + continuation: { ...continuationRun, ...lockChanged, finishedAt: minutesAgo(0.4), ageSeconds: 30 }, + lockOwner: "followUp", expected: { state: "working", canRetry: false, observed: "followUp" } }, + { name: "native continuation cancelled because the native follow-up turn took the lock", + followUp: { status: "running", startedAt: minutesAgo(0.5), ageSeconds: 100 }, + continuation: { ...continuationRun, ...lockChanged, finishedAt: minutesAgo(0.4), ageSeconds: 30 }, + lockOwner: "followUp", native: true, expected: { state: "working", canRetry: false, observed: "followUp" } }, + { name: "continuation cancelled by a lock transfer while the lock pointer is between owners", + followUp: { status: "running", startedAt: minutesAgo(0.5), ageSeconds: 100 }, + continuation: { ...continuationRun, ...lockChanged, finishedAt: minutesAgo(0.4), ageSeconds: 30 }, + lockOwner: null, expected: { state: "working", canRetry: false, observed: "followUp" } }, + { name: "overdue scheduled continuation behind a running lock owner", + followUp: { status: "running", startedAt: minutesAgo(10), ageSeconds: 900 }, + continuation: { ...continuationRun, status: "scheduled_retry", scheduledRetryAt: minutesAgo(5), ageSeconds: 600 }, + lockOwner: "followUp", expected: { state: "working", canRetry: false, observed: "followUp" } }, + { name: "newer failed attempt while an older queued run owns the lock", + followUp: { status: "queued", ageSeconds: 900 }, + continuation: { status: "failed", errorCode: "adapter_failed", startedAt: minutesAgo(3), finishedAt: minutesAgo(1), ageSeconds: 200 }, + lockOwner: "followUp", expected: { state: "working", canRetry: false, observed: "followUp" }, retryWith: "continuation" }, + { name: "continuation queued past its move-forward wait behind a newer failed attempt", + followUp: { status: "queued", ageSeconds: 1200 }, + continuation: { status: "failed", errorCode: "adapter_failed", startedAt: minutesAgo(3), finishedAt: minutesAgo(1), ageSeconds: 200 }, + lockOwner: "followUp", deliveredContinuation: true, + expected: { state: "working", canRetry: true, observed: "followUp", retryAction: "move_forward" } }, + { name: "follow-up turn finished after superseding the continuation", + followUp: { status: "succeeded", startedAt: minutesAgo(3), finishedAt: minutesAgo(1), ageSeconds: 300 }, + continuation: { ...continuationRun, ...lockChanged, finishedAt: minutesAgo(2.5), ageSeconds: 200 }, + lockOwner: null, expected: { state: "resolved", canRetry: false, observed: "followUp" } }, + { name: "follow-up turn failed after superseding the continuation", + followUp: { status: "failed", errorCode: "adapter_failed", startedAt: minutesAgo(3), finishedAt: minutesAgo(1), ageSeconds: 300 }, + continuation: { ...continuationRun, ...lockChanged, finishedAt: minutesAgo(2.5), ageSeconds: 200 }, + lockOwner: null, expected: { state: "failed", canRetry: true, observed: "followUp" }, retryWith: "followUp" }, + { name: "superseded continuation over a failed predecessor with nothing else to run", + predecessor: { status: "failed", errorCode: "adapter_failed", finishedAt: minutesAgo(1) }, + continuation: { ...continuationRun, ...lockChanged, finishedAt: minutesAgo(0.5), ageSeconds: 30 }, + lockOwner: null, expected: { state: "failed", canRetry: true, observed: "predecessor" }, retryWith: "predecessor" }, + { name: "continuation that started before losing the lock is still a failed attempt", + continuation: { ...continuationRun, ...lockChanged, startedAt: minutesAgo(1), finishedAt: minutesAgo(0.5), ageSeconds: 90 }, + lockOwner: null, expected: { state: "failed", canRetry: true, observed: "continuation" }, retryWith: "continuation" }, + { name: "a lock pointer left on a finished run does not hide its failure", + followUp: { status: "failed", errorCode: "adapter_failed", startedAt: minutesAgo(3), finishedAt: minutesAgo(1), ageSeconds: 300 }, + lockOwner: "followUp", expected: { state: "failed", canRetry: true, observed: "followUp" }, retryWith: "followUp" }, +]; + +describe.skipIf(!support.supported)("Retry banner while another run owns the task (restart-chaos t032)", () => { + let temporary: Awaited>; + let db: ReturnType; + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("retry-banner-execution-owner-"); + db = createDb(temporary.connectionString); + }); + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + async function seed(row: Row) { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Owner task", issuePrefix: `O${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Owner agent", adapterType: "process" }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board-user", + status: "active", membershipRole: "member" }); + const [issue] = await db.insert(issues).values({ companyId, title: "Staging mix parent", status: "in_progress", + assigneeAgentId: agentId, createdByUserId: "board-user" }).returning(); + const ids = { predecessor: randomUUID(), followUp: randomUUID(), continuation: randomUUID() }; + const base = { companyId, agentId, scopeKind: "issue" as const, contextSnapshot: { issueId: issue!.id }, + ...(row.native ? { nativeIssueId: issue!.id, runtimeMode: "native" } : { issueId: issue!.id }) }; + await db.insert(heartbeatRuns).values({ ...base, id: ids.predecessor, status: "succeeded", livenessState: "advanced", + createdAt: minutesAgo(30), startedAt: minutesAgo(30), finishedAt: minutesAgo(2), ...row.predecessor }); + for (const key of ["followUp", "continuation"] as const) { + const spec = row[key]; + if (!spec) continue; + const { ageSeconds, ...values } = spec; + await db.insert(heartbeatRuns).values({ ...base, id: ids[key], createdAt: new Date(Date.now() - ageSeconds * 1000), + ...(key === "continuation" ? { retryOfRunId: ids.predecessor } : {}), ...values }); + } + if (row.deliveredContinuation) await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issue!.id, + kind: "active_run_watchdog", status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: agentId, + cause: "fresh_continuation", fingerprint: `fresh-continuation:${ids.predecessor}`, nextAction: "Continuing.", + resolvedAt: minutesAgo(19), attemptCount: 1, evidence: { runId: ids.predecessor, continuationDelivery: "delivered", + continuationRunId: ids.followUp, freshContinuation: { sourceRunId: ids.predecessor, recordedAt: minutesAgo(19).toISOString() } } }); + const [current] = await db.update(issues).set({ executionRunId: row.lockOwner ? ids[row.lockOwner] : null }) + .where(eq(issues.id, issue!.id)).returning(); + return { companyId, issue: current!, ids }; + } + + it.each(STATE_TABLE)("$name", async row => { + const f = await seed(row); + const list = (await taskInterruptionPresentations(db, f.companyId, [f.issue])).get(f.issue.id); + const detail = await taskInterruptionPresentation(db, f.issue); + for (const presentation of [list, detail]) { + expect(presentation).toMatchObject({ state: row.expected.state, canRetry: row.expected.canRetry, + observedRunId: f.ids[row.expected.observed], ...(row.expected.retryAction ? { retryAction: row.expected.retryAction } : {}) }); + expect(presentation?.retryAction).toBe(row.expected.retryAction); + } + if (row.expected.canRetry) { + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: "board-user", + actorSource: "session", idempotencyKey: randomUUID(), observedRunId: list!.observedRunId, + ...(list!.retryAction ? { retryAction: list!.retryAction } : {}) }); + expect(receipt).toMatchObject({ accepted: true, stale: false, observedRunId: list!.observedRunId }); + } + if (row.lockOwner && row.expected.observed === row.lockOwner && !row.expected.canRetry) { + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: "board-user", + actorSource: "session", idempotencyKey: randomUUID(), observedRunId: f.ids[row.lockOwner] }); + expect(receipt).toMatchObject({ accepted: true, stale: false, observedRunId: f.ids[row.lockOwner] }); + } + if (row.retryWith && !row.expected.canRetry) { + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: "board-user", + actorSource: "session", idempotencyKey: randomUUID(), observedRunId: f.ids[row.retryWith] }); + expect(receipt).toMatchObject({ accepted: true, stale: false, observedRunId: f.ids[row.retryWith] }); + } + }, 60_000); + + describe("a held Retry beside an unstarted lock-changed cancellation", () => { + async function seedHeldRetry(newer: "lock_changed" | "failed") { + const row: Row = { name: "held", lockOwner: null, + predecessor: { status: "failed", errorCode: "adapter_failed", finishedAt: minutesAgo(1) }, + continuation: newer === "lock_changed" + ? { ...continuationRun, ...lockChanged, finishedAt: minutesAgo(0.5), ageSeconds: 30 } + : { status: "failed", errorCode: "adapter_failed", startedAt: minutesAgo(0.8), finishedAt: minutesAgo(0.5), ageSeconds: 50 }, + expected: { state: "failed", canRetry: true, observed: "predecessor" } }; + const f = await seed(row); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.issue.assigneeAgentId!, + source: "on_demand", reason: "task_retry", status: "deferred_issue_execution", requestedByActorType: "user", + requestedByActorId: "board-user", payload: { issueId: f.issue.id, taskRetry: { sourceRunId: f.ids.predecessor }, + executionWait: { reason: "process_running" } } }).returning(); + return { ...f, requestId: wake!.id }; + } + + it("keeps the held Retry's refusal current and does not retire the Retry", async () => { + const f = await seedHeldRetry("lock_changed"); + expect((await heldTaskRetryRefusals(db, f.companyId, [f.issue.id])).get(f.issue.id)).toBe("process_running"); + await expect(db.transaction(tx => retireSupersededTaskRetry(tx as unknown as typeof db, { companyId: f.companyId, + issueId: f.issue.id, requestId: f.requestId, sourceRunId: f.ids.predecessor }))).resolves.toBe(false); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.requestId)); + expect(wake).toMatchObject({ status: "deferred_issue_execution", error: null }); + }, 60_000); + + it("still retires a held Retry that a newer real attempt superseded", async () => { + const f = await seedHeldRetry("failed"); + expect((await heldTaskRetryRefusals(db, f.companyId, [f.issue.id])).has(f.issue.id)).toBe(false); + await expect(db.transaction(tx => retireSupersededTaskRetry(tx as unknown as typeof db, { companyId: f.companyId, + issueId: f.issue.id, requestId: f.requestId, sourceRunId: f.ids.predecessor }))).resolves.toBe(true); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.requestId)); + expect(wake).toMatchObject({ status: "skipped", error: "task_retry_superseded" }); + }, 60_000); + }); +}); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 283a9075c4f..9ed2a774f09 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -2,7 +2,7 @@ import { ENVIRONMENT_PROVIDER_WAIT_KEY, ENVIRONMENT_PROVIDER_WAIT_RETRY_REASON, import { STORAGE_UNAVAILABLE_WAIT_RETRY_REASON } from "../storage/storage-unavailable.js"; import { AGENT_DIRECTORY_WAIT_RETRY_REASON, INTERRUPTED_PREPARATION_RETRY_REASON } from "./execution-recovery-attempt.js"; import { boundSetupCommandFailure, readSetupCommandFailure } from "./workspace-setup-command-failure.js"; -import { and, desc, eq, inArray, isNull, not, notInArray, sql, type SQL } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, not, notInArray, or, sql, type SQL } from "drizzle-orm"; import { unidentifiedLocalLaunchRefusals } from "./native-local-process-stop.js"; import { nativeSessionCleanupBlock } from "../vendor/paperclip-runner/index.js"; import { agents, agentWakeupRequests, approvals, chatActions, companies, environmentLeases, heartbeatRuns, issueApprovals, issueRecoveryActions, @@ -26,7 +26,7 @@ import { canonicalExecutionUuid } from "./execution-identity.js"; import { chatActionDeadline } from "./chat-action-work.js"; import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED } from "./native-runtime/completed-turn-cleanup.js"; import { isAcknowledgedLegacyUserStop, isQueuedDependencyWait, runStopPresentationJson } from "./run-cancellation.js"; -import { heldRetryOffersRetry } from "./task-retry-eligibility.js"; +import { heldRetryOffersRetry, liveExecutionOwnerCondition, taskAttemptCondition } from "./task-retry-eligibility.js"; import { exhaustedNativeFinalizationCondition, savedResultRetryPendingCondition } from "./native-runtime/exhausted-finalization-retry.js"; import { AI_CONNECTION_BUSY_RETRY_REASON, AI_CONNECTION_POOL_WAIT_RETRY_REASON, RUNNER_REVISION_WAIT_RETRY_REASON, WORKSPACE_BUSY_RETRY_REASON } from "../modules/run-dispatch/index.js"; @@ -457,18 +457,21 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i const requestedUuid = sql`case when ${requestedIssueId} ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' then ${requestedIssueId}::uuid else null end`; const legacyIssue = sql`${heartbeatRuns.contextSnapshot}->>'issueId'`; - const attempt = sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null - and coalesce(${heartbeatRuns.errorCode}, '') in ('recovery_no_longer_needed', 'issue_closed_before_start'))`; - const candidate = (condition: ReturnType) => db.select({ id: heartbeatRuns.id, createdAt: heartbeatRuns.createdAt }) - .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), condition, attempt)) + const candidate = (condition: SQL | undefined, executionOwner = false) => db.select({ id: heartbeatRuns.id, + createdAt: heartbeatRuns.createdAt, executionOwner: sql`${executionOwner ? 1 : 0}::int`.as("execution_owner") }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), condition, taskAttemptCondition())) .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + const nativeRun = eq(heartbeatRuns.nativeIssueId, requestedUuid); + const firstClassRun = and(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.issueId, requestedUuid)); + const legacyRun = and(isNull(heartbeatRuns.nativeIssueId), isNull(heartbeatRuns.issueId), eq(legacyIssue, requestedIssueId)); const candidates = unionAll( - candidate(eq(heartbeatRuns.nativeIssueId, requestedUuid)), - candidate(and(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.issueId, requestedUuid))), - candidate(and(isNull(heartbeatRuns.nativeIssueId), isNull(heartbeatRuns.issueId), eq(legacyIssue, requestedIssueId))), + candidate(and(liveExecutionOwnerCondition(companyId, requestedUuid), or(nativeRun, firstClassRun, legacyRun)), true), + candidate(nativeRun), + candidate(firstClassRun), + candidate(legacyRun), ).as("interruption_run_candidates"); const latestRuns = db.select({ id: candidates.id }).from(candidates) - .orderBy(desc(candidates.createdAt), desc(candidates.id)).limit(1).as("latest_interruption_runs"); + .orderBy(desc(candidates.executionOwner), desc(candidates.createdAt), desc(candidates.id)).limit(1).as("latest_interruption_runs"); const wakeIssue = sql`${agentWakeupRequests.payload}->>'issueId'`; const isRetryWait = sql`coalesce(jsonb_typeof(${agentWakeupRequests.payload}#>'{taskRetry,sourceRunId}') = 'string', false)`; const agentIds = selected.flatMap(issue => issue.assigneeAgentId ? [issue.assigneeAgentId] : []); diff --git a/server/src/services/task-retry-eligibility.ts b/server/src/services/task-retry-eligibility.ts index e4e3ef1719b..410bd902b07 100644 --- a/server/src/services/task-retry-eligibility.ts +++ b/server/src/services/task-retry-eligibility.ts @@ -1,5 +1,18 @@ -import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; -import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; +import { and, desc, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; +import { agentWakeupRequests, heartbeatRuns, issues, type Db } from "@paperclipai/db"; + +const UNSTARTED_SUPERSEDED_RUN_CODES = ["recovery_no_longer_needed", "issue_closed_before_start", "issue_execution_lock_changed"]; + +export function taskAttemptCondition(run: { status: SQL | typeof heartbeatRuns.status; startedAt: SQL | typeof heartbeatRuns.startedAt; + errorCode: SQL | typeof heartbeatRuns.errorCode } = heartbeatRuns) { + return sql`not (${run.status} = 'cancelled' and ${run.startedAt} is null + and coalesce(${run.errorCode}, '') in (${sql.join(UNSTARTED_SUPERSEDED_RUN_CODES.map(code => sql`${code}`), sql`, `)}))`; +} + +export function liveExecutionOwnerCondition(companyId: string, issueId: SQL | string) { + return sql`${heartbeatRuns.status} in ('queued', 'running') and ${heartbeatRuns.id} = (select ${issues.executionRunId} from ${issues} + where ${issues.companyId} = ${companyId} and ${issues.id} = ${issueId})`; +} const REFUSALS_A_CLICK_CANNOT_CHANGE: ReadonlySet = new Set([ "cancelled_by_operator", "retry_source_changed", "workspace_owner_stop_required", "previous_execution_unverified", @@ -19,6 +32,7 @@ export async function heldTaskRetryRefusals(db: Pick, co where latest.company_id = ${companyId} and (latest.issue_id::text = ${agentWakeupRequests.payload}->>'issueId' or latest.native_issue_id::text = ${agentWakeupRequests.payload}->>'issueId' or latest.context_snapshot->>'issueId' = ${agentWakeupRequests.payload}->>'issueId') + and ${taskAttemptCondition({ status: sql`latest.status`, startedAt: sql`latest.started_at`, errorCode: sql`latest.error_code` })} order by latest.created_at desc, latest.id desc limit 1)` }) .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index a060fa14ea8..486466d4d46 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -23,6 +23,7 @@ import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state import { deferredChatInputPredicate, resumeDeferredChatWake, unadmittedChatWakeupCondition } from "./durable-chat-wakeup.js"; import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; import { hasPendingRunCleanup } from "./execution-blocker.js"; +import { liveExecutionOwnerCondition, taskAttemptCondition } from "./task-retry-eligibility.js"; import { undeliveredContinuation } from "./fresh-continuation-retry-fold.js"; import { PersistedExecutionWaitRefusal } from "./deferred-execution-wait.js"; import { rearmExhaustedNativeFinalization } from "./native-runtime/exhausted-finalization-retry.js"; @@ -203,7 +204,7 @@ export async function retireSupersededTaskRetry(db: Db, input: { const [latest] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.companyId), or(eq(heartbeatRuns.issueId, input.issueId), eq(heartbeatRuns.nativeIssueId, input.issueId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${input.issueId}`), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${input.issueId}`), taskAttemptCondition(), )).orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); if (latest?.id === input.sourceRunId) return false; const [wake] = await db.select({ agentId: agentWakeupRequests.agentId }).from(agentWakeupRequests).where(and( @@ -311,8 +312,10 @@ function admitTaskRetry(input: TaskRetryRequest) { } const [latest] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, issue.companyId), or(eq(heartbeatRuns.issueId, issue.id), eq(heartbeatRuns.nativeIssueId, issue.id), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`))) - .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`), taskAttemptCondition())) + .orderBy(...(input.observedRunId ? [sql`case when ${heartbeatRuns.id}::text = ${input.observedRunId} + and ${liveExecutionOwnerCondition(issue.companyId, issue.id)} then 0 else 1 end`] : []), + desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); if (["done", "cancelled"].includes(issue.status) || !issue.assigneeAgentId || !latest || latest.agentId !== issue.assigneeAgentId || input.observedRunId !== latest.id) { return { requestId: null, accepted: false, stale: true, observedRunId: latest?.id ?? null, chatActionId: undefined }; @@ -542,7 +545,7 @@ async function continueRefusedTaskChatRetry(db: Db, wake: typeof agentWakeupRequ const [latest] = await tx.select({ id: heartbeatRuns.id, finishedAt: heartbeatRuns.finishedAt, status: heartbeatRuns.status, startedAt: heartbeatRuns.startedAt, errorCode: heartbeatRuns.errorCode, resultJson: heartbeatRuns.resultJson }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, wake.companyId), or(eq(heartbeatRuns.issueId, issueId), eq(heartbeatRuns.nativeIssueId, issueId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`))) + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`), taskAttemptCondition())) .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); if (latest?.id !== intent.sourceRunId || !latest.finishedAt) return false; if (intent.initiatedBy) {