diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 8422a92ce12..1cde5010c5c 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1787,6 +1787,54 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }, ); + it("routes a credential sharing denial to its owner when the controller lease lapses during AI selection", async () => { + const { companyId, agentId, issueId, runId } = await seedQueuedIssueRunFixture(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "responsible-user", status: "active", membershipRole: "member" }).onConflictDoNothing(); + await db.update(agents).set({ runtimeConfig: { + heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 }, + aiConnection: { provider: "openai", method: "api_key", mode: "responsible_user" }, + } }).where(eq(agents.id, agentId)); + let runController: AbortController | undefined; + const originalWatch = controllerLeases.watchLegacyControllerLease; + const watcher = vi.spyOn(controllerLeases, "watchLegacyControllerLease").mockImplementation((...args) => { + if (args[1].id === runId) runController = args[2]; + return originalWatch(...args); + }); + // The renewal deadline passes while selection reads the grant, then the + // selection's own sharing denial reaches the setup failure path. + const prepare = vi.spyOn(aiConnectionRuntime, "prepareManagedAiRuntime").mockImplementationOnce(async () => { + runController!.abort(new controllerLeases.LegacyControllerLeaseLostError("renewal_deadline", + { sinceRenewalMs: 37_725, renewalPendingMs: 29_418, deadlineLateMs: 52, renewalCount: 2 })); + throw aiConnectionCredentialNotSharedFailure(); + }); + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + await heartbeat.waitForRunExecutionDrain(runId); + await waitForPendingRunFailureReports(); + expect(prepare).toHaveBeenCalledOnce(); + expect(await heartbeat.getRun(runId)).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", + error: "This credential is not shared with the responsible user", resultJson: { + configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_credential_not_shared" }, + controllerLeaseLoss: { cause: "renewal_deadline", renewalPendingMs: 29_418 }, + executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + expect(mockAdapterExecute).not.toHaveBeenCalled(); + // No retry that would repeat the same denial before the owner acts. + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(issue).toMatchObject({ status: "blocked", executionRunId: null }); + const [action] = await selectRecoveryActionsForIssue(issueId); + expect(action).toMatchObject({ status: "active", kind: "configuration_validation", cause: "configuration_incomplete", ownerType: "board" }); + } finally { + await heartbeat.drainActiveRunExecutions().catch(() => undefined); + prepare.mockRestore(); + watcher.mockRestore(); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + } + }); + it.each(["timeout", "upstream", "cleanup_pending", "cleanup_pending_edited", "cleanup_pending_exhausted", "edited", "exhausted", "new_message", "no_claim", "reassigned", "superseded", "stopped"])( "settles explicit retry admission through real executor cleanup: %s", async scenario => { const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); diff --git a/server/src/lib/ai-connection-configuration-failure.ts b/server/src/lib/ai-connection-configuration-failure.ts index 5acb51c6d62..28bdc0da2d7 100644 --- a/server/src/lib/ai-connection-configuration-failure.ts +++ b/server/src/lib/ai-connection-configuration-failure.ts @@ -22,3 +22,14 @@ export function readAiConnectionConfigurationFailure(error: unknown): AiConnecti export function isAiConnectionConfigurationReason(value: unknown): value is AiConnectionConfigurationReason { return typeof value === "string" && reasons.some((reason) => reason === value); } + +/** + * A run's `configurationIncomplete` result records an AI selection that an owned + * producer rejected, not the preparation step's broad catch wrapper. The same + * selection fails again on any host until its owner acts. + */ +export function isOwnedAiConnectionSelectionRejection(configurationIncomplete: unknown): boolean { + if (configurationIncomplete === null || typeof configurationIncomplete !== "object" || Array.isArray(configurationIncomplete)) return false; + const configuration = configurationIncomplete as Record; + return configuration.reason === "ai_connection_unavailable" && isAiConnectionConfigurationReason(configuration.selectionFailure); +} diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index 9e288030a26..ce6a396bd58 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -724,6 +724,20 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(run).toEqual(saved); }); + it.each(["configuration_incomplete", "controller_lease_lost"] as const)( + "decides a credential denial that coincided with a lease loss by its recorded code: %s", async (errorCode) => { + await seedCompanyAndAgent(); + const run = aiSelectionRun({ errorCode, error: "This credential is not shared with the responsible user" }); + Object.assign(run.resultJson!, { + controllerLeaseLoss: { cause: "renewal_deadline", renewalCount: 2, renewalPendingMs: 29_418 }, + cancellation: { source: "controller_lease", expected: false, reason: "Legacy controller lease lost (renewal_deadline)" }, + }); + (run.resultJson!.configurationIncomplete as Record).selectionFailure = "ai_connection_credential_not_shared"; + await reportRunFailure(db, run, { phase: "setup" }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(errorCode === "controller_lease_lost" ? 1 : 0); + }, + ); + it.each([undefined, null, {}, [], "", "database_error", "ai_connection_busy"])( "reports AI connection wrappers without a known selection cause: %j", async (selectionFailure) => { await seedCompanyAndAgent(); diff --git a/server/src/services/ai-connection-configuration-failure.ts b/server/src/services/ai-connection-configuration-failure.ts index a31275b3185..b59214d2ed9 100644 --- a/server/src/services/ai-connection-configuration-failure.ts +++ b/server/src/services/ai-connection-configuration-failure.ts @@ -1,6 +1,6 @@ import { forbidden, unprocessable } from "../errors.js"; import { recordAiConnectionConfigurationFailure, type AiConnectionConfigurationReason } from "../lib/ai-connection-configuration-failure.js"; -export { readAiConnectionConfigurationFailure, isAiConnectionConfigurationReason } from "../lib/ai-connection-configuration-failure.js"; +export { readAiConnectionConfigurationFailure, isAiConnectionConfigurationReason, isOwnedAiConnectionSelectionRejection } from "../lib/ai-connection-configuration-failure.js"; type CredentialAccessFailure = { connectionName: string; grantId?: string }; const credentialAccessFailures = new WeakMap(); diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 219b630feaa..a2f603c9ea4 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -1865,6 +1865,23 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( resultJson: { cancellation: { source: "controller_lease", reason: "Legacy controller lease lost (renewal_deadline)" } } }); }); + it.each([ + { name: "a rejected AI selection keeps its owner action", selectionFailure: "ai_connection_credential_not_shared", errorCode: "configuration_incomplete" }, + { name: "an unclassified AI preparation error stays a retryable lease loss", selectionFailure: undefined, errorCode: "controller_lease_lost" }, + ] as const)("records a lease lost during AI selection: $name", async ({ selectionFailure, errorCode }) => { + const f = await fixture(); + f.controller.abort(new LegacyControllerLeaseLostError("renewal_deadline", + { sinceRenewalMs: 37_725, renewalPendingMs: 29_418, deadlineLateMs: 52, renewalCount: 2 })); + await f.completion.failRunSetup(f.setup(new ConfigurationIncompleteFailure("This credential is not shared with the responsible user", { + configurationIncomplete: { reason: "ai_connection_unavailable", ...(selectionFailure ? { selectionFailure } : {}) } }))); + expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode, + error: "This credential is not shared with the responsible user", resultJson: { + configurationIncomplete: { reason: "ai_connection_unavailable" }, + executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + controllerLeaseLoss: { cause: "renewal_deadline", renewalPendingMs: 29_418, renewalCount: 2 }, + cancellation: { source: "controller_lease", reason: "Legacy controller lease lost (renewal_deadline)" } } }); + }); + it("does not make a lease loss retryable once provider work has started", async () => { const f = await fixture(); f.controller.abort(new LegacyControllerLeaseLostError("renewal_error")); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index c76c9010cec..67b57d35159 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -34,6 +34,7 @@ import { isWorkspaceValidationFailedRun, isWorkspaceValidationFailure, attachPap import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js"; import { cancellationResultJson, controllerLeaseLossCancellation, readRunCancellation } from "../run-cancellation.js"; import { CONTROLLER_LEASE_LOST_FAILURE_CODE, LegacyControllerLeaseLostError } from "../legacy-controller-lease.js"; +import { isOwnedAiConnectionSelectionRejection } from "../ai-connection-configuration-failure.js"; import { REVIEWED_CHAT_BINDING_NOT_READY, ReviewedChatBindingNotReadyError, reviewedChatBindingDenialResultJson } from "../reviewed-chat-binding-diagnostics.js"; import { reportRunFailure } from "../run-failure-report.js"; import { accountRunCost } from "../run-cost-accounting.js"; @@ -1674,8 +1675,12 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC ? outerErr : null; const runnerRevisionMismatch = !providerDispatchStarted && run.runtimeMode === "legacy" && run.nativeSessionId === null && run.runnerInstanceId === null && isRunnerRevisionMismatch(outerErr); + // A lease lost while AI selection rejected the account does not make the + // run retryable: every host repeats that rejection until its owner acts. + const ownedAiSelectionRejection = isOwnedAiConnectionSelectionRejection( + configurationIncompleteSetupFailure?.resultJson.configurationIncomplete); const setupFailureErrorCode = - (controllerLeaseLost ? CONTROLLER_LEASE_LOST_FAILURE_CODE : null) ?? + (controllerLeaseLost && !ownedAiSelectionRejection ? CONTROLLER_LEASE_LOST_FAILURE_CODE : null) ?? (storageUnavailable ? STORAGE_UNAVAILABLE_CODE : null) ?? remoteWorkspacePreparationTimeout?.code ?? reusableSandboxResumeFailure?.code ?? diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index e4592e70901..0e6314c7c58 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -14,7 +14,7 @@ import { import { logger } from "../middleware/logger.js"; import { isUnexpectedRunCancellation } from "./run-cancellation.js"; import { readConnectionFailure, type ConnectionFailure } from "@paperclipai/adapter-utils/connection-failure"; -import { isAiConnectionConfigurationReason } from "./ai-connection-configuration-failure.js"; +import { isOwnedAiConnectionSelectionRejection } from "./ai-connection-configuration-failure.js"; import { readRunFailureReportingMode, type RunFailureReportingMode } from "../sentry-run-failure-policy.js"; import { RESTRICTED_WORKSPACE_SOURCE_REASON } from "./workspace-source-restriction.js"; import { PLUGIN_RUNNER_AI_CONNECTION_REQUIRED } from "./plugin-runner-ai-connection.js"; @@ -88,10 +88,8 @@ function isAiConnectionSelectionBlocker(run: HeartbeatRun, options: RunFailureRe run.executionStage !== "preparing" || options.phase !== "setup" || run.exitCode != null || run.signal != null ) return false; - const configuration = asRecord(run.resultJson?.configurationIncomplete); const recovery = asRecord(run.resultJson?.executionRecovery); - return configuration?.reason === "ai_connection_unavailable" && - isAiConnectionConfigurationReason(configuration.selectionFailure) && + return isOwnedAiConnectionSelectionRejection(run.resultJson?.configurationIncomplete) && recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false; }