diff --git a/doc/project-environment-compatibility.md b/doc/project-environment-compatibility.md index 39ac8854044..65e1807fc63 100644 --- a/doc/project-environment-compatibility.md +++ b/doc/project-environment-compatibility.md @@ -73,6 +73,13 @@ projects keep their stored owner. This change does not migrate managed storage or detach projects that contain existing work. A managed project run in an incompatible environment fails with `project_environment_incompatible` before a workspace provider operation, with a message naming the storage and environment. +On a volume-free server, execution admission refuses that run earlier. When the +task source's project storage cannot serve the selected environment, the run +fails `configuration_incomplete` with reason +`project_storage_environment_incompatible` and the same message. The task notice +names the owner action, and run-failure reporting does not send it to Sentry. +A task that moves to another project keeps its selected source, so the source +project's storage still decides compatibility. ## Project Configuration diff --git a/doc/workspace-storage.md b/doc/workspace-storage.md index b18e9338884..92dc535f532 100644 --- a/doc/workspace-storage.md +++ b/doc/workspace-storage.md @@ -363,6 +363,11 @@ selector checks the resolved native warm lifecycle before admission and rejects legacy or per-turn execution. External conversation providers keep their existing exemption because they own no task filesystem. Source resolution checks this contract again before a configured host folder can fall back to local storage. +When the task source's project uses plugin storage that the selected +environment cannot serve, the refusal names that storage and environment and +ends the run `configuration_incomplete` (reason +`project_storage_environment_incompatible`). See +[project-environment-compatibility.md](project-environment-compatibility.md). A replacement controller reconstructs an active remote agent-directory cache from the registered sandbox, including unsaved files. The remote directory, diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts index ce6a396bd58..01a7dae4263 100644 --- a/server/src/services/__tests__/run-failure-report.test.ts +++ b/server/src/services/__tests__/run-failure-report.test.ts @@ -949,6 +949,34 @@ describeEmbeddedPostgres("reportRunFailure", () => { expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); }); + function projectStorageEnvironmentRun(overrides: Partial = {}) { + return buildRun({ + errorCode: "configuration_incomplete", + executionStage: "preparing", + resultJson: { + configurationIncomplete: { reason: "project_storage_environment_incompatible", actionUrl: "/agents/fixture/runtime" }, + executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + }, + ...overrides, + }); + } + + it("keeps a run refused for incompatible project workspace storage actionable on the task", async () => { + await seedCompanyAndAgent(); + await reportRunFailure(db, projectStorageEnvironmentRun(), { phase: "setup" }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + + it.each([ + { executionStage: "executing" }, + { errorCode: "setup_failed" }, + { resultJson: { configurationIncomplete: { reason: "project_storage_environment_incompatible" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } }, + ])("reports project storage markers without an exact pre-dispatch outcome: %j", async (overrides) => { + await seedCompanyAndAgent(); + await reportRunFailure(db, projectStorageEnvironmentRun(overrides), { phase: "setup" }); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + }); + function localPathConfigurationRun(overrides: Partial = {}) { return buildRun({ errorCode: "workspace_validation_failed", diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index f3966a264a6..4bb60c0e667 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -38,6 +38,7 @@ import { buildProjectMentionHref, readRemoteWorkspaceAuthority } from "@papercli import { migrateRemoteWorkspace } from "../remote-workspace-migration.js"; import { trace } from "@opentelemetry/api"; import { effectiveProjectWorkspacePolicy } from "../project-workspace-provider.js"; +import { createPluginWorkspaceFixture } from "../../__tests__/helpers/plugin-workspaces.js"; import { resolveExecutionWorkspaceMode, buildExecutionWorkspaceAdapterConfig } from "../execution-workspace-policy.js"; import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; import { createStorageService } from "../../storage/service.js"; @@ -633,6 +634,33 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace expect(acquire).not.toHaveBeenCalled(); }); + it("names an incompatible project storage environment when a volume-free server rejects the run", async () => { + const { input, envOrchestrator } = await fixture(); + const storage = await createPluginWorkspaceFixture(db, input.agent.companyId); + input.agent.adapterType = "paperclip_runner"; + input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment, + name: "Daytona", driver: "sandbox", config: { provider: "daytona" } }; + input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true; + configureRemotePersistence(); + process.env.PAPERCLIP_EPHEMERAL_SERVER = "true"; + const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun"); + const acquire = vi.spyOn(envOrchestrator, "acquireForRun"); + input.policy.projectExecutionWorkspacePolicy = { enabled: true, storageProviderPluginId: storage.pluginId }; + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ name: "ConfigurationIncompleteFailure", + code: "configuration_incomplete", + message: `This project's workspace storage (Storage) does not support the environment "Daytona". Use an agent in a compatible environment, or create a project for this environment.`, + resultJson: { configurationIncomplete: { reason: "project_storage_environment_incompatible", + storageProviderPluginId: storage.pluginId, environmentId: input.environment.selectedEnvironmentForConfig.id, + fingerprint: `project_storage_environment:${storage.pluginId}:${input.environment.selectedEnvironmentForConfig.id}` } } }); + // An unavailable provider cannot explain the rejection; keep the original reason. + input.policy.projectExecutionWorkspacePolicy = { enabled: true, storageProviderPluginId: randomUUID() }; + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow( + "Volume-free servers require native Paperclip Runner execution with remote workspace storage."); + expect(resolve).not.toHaveBeenCalled(); + expect(acquire).not.toHaveBeenCalled(); + expect(await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.companyId, input.agent.companyId))).toHaveLength(0); + }); + it.each(["configured-folder", "explicit-task-directory", "enrolled-remote"] as const)( "preserves effective configured cwd authority for %s", async scenario => { const { input, source, issue, envOrchestrator } = await fixture(); diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index 851da8f5724..b050796c038 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -2,6 +2,9 @@ import { NATIVE_REMOTE_RECOVERY_LEASE_MISMATCH, remoteRecoveryLeaseMismatch } fr import { readInheritedRunnerRetainedSandbox } from "../native-runtime/native-inherited-runner-retry.js"; import { resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js"; import { assertEphemeralWorkspaceExecution, readRemoteWorkspaceAuthority, RemoteWorkspaceWriterActiveError, reserveRemoteWorkspace, shouldUseRemoteWorkspaces } from "../remote-workspaces.js"; +import { assertProjectStorageEnvironment, listProjectWorkspaceProviders } from "../project-workspace-provider.js"; +import { PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE } from "../project-storage-environment-reason.js"; +import { ConfigurationIncompleteFailure } from "./run-preparation.js"; import { WorkspaceBusyDeferral } from "./retries.js"; import { remoteWorkspaceHoldRefusesStart } from "../remote-workspace-writer-hold.js"; import { WORKSPACE_BUSY_RETRY_REASON } from "../../modules/run-dispatch/index.js"; @@ -283,10 +286,35 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace && !projectExecutionWorkspacePolicy?.storageProviderPluginId, existingWorkspace: reusableExistingExecutionWorkspace ?? existingExecutionWorkspace, }); - if (!providerApiPlacement) assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, - adapterType: agent.adapterType, - environmentDriver: selectedEnvironmentForConfig?.driver, - pluginTaskWorkspace, remoteAuthority }); + // A volume-free server rejects execution without durable task storage. When + // the source project's storage provider cannot serve the selected + // environment, that mismatch is the cause and only an owner can repair it. + const assertEphemeralExecution = async () => { + if (providerApiPlacement) return; + try { + assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, adapterType: agent.adapterType, + environmentDriver: selectedEnvironmentForConfig?.driver, pluginTaskWorkspace, remoteAuthority }); + } catch (error) { + const storagePluginId = projectExecutionWorkspacePolicy?.storageProviderPluginId; + const provider = storagePluginId && selectedEnvironmentForConfig + ? (await listProjectWorkspaceProviders(db, agent.companyId)).find(value => value.id === storagePluginId) : undefined; + if (provider && selectedEnvironmentForConfig) { + try { + assertProjectStorageEnvironment(provider, selectedEnvironmentForConfig); + } catch (mismatch) { + throw new ConfigurationIncompleteFailure(mismatch instanceof Error ? mismatch.message : String(mismatch), { + configurationIncomplete: { reason: PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE, companyId: agent.companyId, + agentId: agent.id, issueId: issueId ?? null, projectId: issueRef?.projectId ?? null, + storageProviderPluginId: provider.id, environmentId: selectedEnvironmentForConfig.id, + fingerprint: `project_storage_environment:${provider.id}:${selectedEnvironmentForConfig.id}`, + actionUrl: `/agents/${agent.id}/runtime`, missingBindings: [] }, + }); + } + } + throw error; + } + }; + await assertEphemeralExecution(); const assertRemotePreparationOwned = run.runtimeMode === "native" ? () => assertNativeSessionPreparationAuthority({ db, companyId: agent.companyId, runId: run.id, leaseOwner: runOptions.nativeLeaseOwner ?? runOptions.nativeRestartRecovery?.leaseOwner, @@ -370,8 +398,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace remoteAuthority = false; // Source resolution can retain a configured host folder. Check the // resolved authority before materializing it or acquiring a sandbox. - if (!providerApiPlacement) assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, - adapterType: agent.adapterType, environmentDriver: selectedEnvironmentForConfig?.driver, pluginTaskWorkspace, remoteAuthority }); + await assertEphemeralExecution(); } else { remoteSourceState.value = sourceState; diff --git a/server/src/services/project-storage-environment-reason.ts b/server/src/services/project-storage-environment-reason.ts new file mode 100644 index 00000000000..9848202d79e --- /dev/null +++ b/server/src/services/project-storage-environment-reason.ts @@ -0,0 +1,5 @@ +// Dependency-free so run-failure reporting and recovery notices can classify +// this outcome without loading the plugin and workspace graph. + +/** The task source's project storage provider cannot serve the run's environment. */ +export const PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE = "project_storage_environment_incompatible"; diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 513d1b1ed10..ffdabb24fe3 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -24,6 +24,7 @@ import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; import { isAiAuthenticationBlocked } from "../ai-auth-failure.js"; import { RESTRICTED_WORKSPACE_SOURCE_REASON } from "../workspace-source-restriction.js"; +import { PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE } from "../project-storage-environment-reason.js"; import { hasCommittedNativePlanWait } from "../native-runtime/native-plan-wait.js"; import { hasCommittedNativeConversationTurnWait, hasCommittedNativeResponseWakeWait } from "../native-runtime/native-response-wake-wait.js"; import { listExternalReviewWaits } from "../native-runtime/external-review-wait.js"; @@ -173,6 +174,7 @@ import { } from "./successful-run-handoff.js"; import { SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON, + PROJECT_STORAGE_ENVIRONMENT_NEXT_ACTION, sandboxProviderPluginRemedy, buildExecutionReviewParticipantRecoveryNoticeSeed, buildExecutionReviewParticipantUnavailableNoticeSeed, @@ -2802,6 +2804,8 @@ export function recoveryService( ? "Reconnect the selected AI account or choose an available connection, then continue the task." : readConfigurationIncompletePayload(input.latestRun)?.reason === RESTRICTED_WORKSPACE_SOURCE_REASON ? "Retry the task." + : readConfigurationIncompletePayload(input.latestRun)?.reason === PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE + ? PROJECT_STORAGE_ENVIRONMENT_NEXT_ACTION : readConfigurationIncompletePayload(input.latestRun) ?.reason === SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON ? `The plugin that starts this agent's sandbox is not running. To fix it, ${sandboxProviderPluginRemedy( diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index 9ed9e3d9eb6..45259f5293d 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -395,6 +395,14 @@ it("names the managed AI connection a plugin Runner-task environment requires", expect(notice.body).not.toContain("secret/env"); }); +it("names incompatible project workspace storage instead of a missing secret", () => { + const notice = buildConfigurationIncompleteRecoveryNoticeSeed({ reason: "project_storage_environment_incompatible" }); + expect(notice.title).toBe("Workspace storage unavailable"); + expect(notice.body).toContain("workspace storage that the agent's environment cannot use"); + expect(notice.nextAction).toContain("Assign the task to an agent whose environment can use"); + expect(notice.body).not.toContain("secret"); +}); + it("tells the operator to switch a preconfigured Runner home to an isolated one", () => { const notice = buildConfigurationIncompleteRecoveryNoticeSeed({ reason: "ai_connection_required", homeMode: "preconfigured" }); expect(notice.title).toBe("Isolated Runner home required"); diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 59da1fac7f8..2fd167dbc5a 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -1,5 +1,6 @@ import type { IssueCommentMetadata, IssueCommentPresentation } from "@paperclipai/shared"; import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js"; +import { PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE } from "../project-storage-environment-reason.js"; import { agentLinkRow, keyValueRow, @@ -128,6 +129,7 @@ export function buildWorkspaceValidationRecoveryNoticeSeed(): StrandedRecoveryNo } export const SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON = "sandbox_provider_plugin_not_ready"; +export const PROJECT_STORAGE_ENVIRONMENT_NEXT_ACTION = "Assign the task to an agent whose environment can use this project's workspace storage, or choose a workspace from a project for this agent's environment, then press Retry."; function readNonEmptyStringField(payload: Record | null | undefined, key: string): string | null { const value = payload?.[key]; @@ -226,6 +228,14 @@ export function buildConfigurationIncompleteRecoveryNoticeSeed( tone: "danger", }; } + if (readNonEmptyStringField(configurationIncomplete, "reason") === PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE) { + return { + title: "Workspace storage unavailable", + body: "The agent did not start because the project that stores this task's files uses workspace storage that the agent's environment cannot use.", + nextAction: PROJECT_STORAGE_ENVIRONMENT_NEXT_ACTION, + tone: "danger", + }; + } if (readNonEmptyStringField(configurationIncomplete, "reason") === SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON) { const pluginKey = readNonEmptyStringField(configurationIncomplete, "pluginKey") ?? "the sandbox provider plugin"; const pluginStatus = readNonEmptyStringField(configurationIncomplete, "pluginStatus") ?? "not ready"; diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index 0e6314c7c58..50d1a074ded 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -18,6 +18,7 @@ import { isOwnedAiConnectionSelectionRejection } from "./ai-connection-configura import { readRunFailureReportingMode, type RunFailureReportingMode } from "../sentry-run-failure-policy.js"; import { RESTRICTED_WORKSPACE_SOURCE_REASON } from "./workspace-source-restriction.js"; import { PLUGIN_RUNNER_AI_CONNECTION_REQUIRED } from "./plugin-runner-ai-connection.js"; +import { PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE } from "./project-storage-environment-reason.js"; import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-runtime/native-provider-failure.js"; import { PROJECT_ACCESS_DENIED } from "./project-access-denial.js"; import { REVIEWED_CHAT_BINDING_NOT_READY } from "./reviewed-chat-binding-diagnostics.js"; @@ -176,6 +177,19 @@ function isPluginRunnerAiConnectionRequiredBlocker(run: HeartbeatRun, options: R recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false; } +/** The source project's storage cannot serve the agent's environment, refused before dispatch; the task notice names the owner action. */ +function isProjectStorageEnvironmentBlocker(run: HeartbeatRun, options: RunFailureReportOptions): boolean { + if ( + run.status !== "failed" || run.errorCode !== "configuration_incomplete" || + run.executionStage !== "preparing" || options.phase !== "setup" || + run.exitCode != null || run.signal != null + ) return false; + const configuration = asRecord(run.resultJson?.configurationIncomplete); + const recovery = asRecord(run.resultJson?.executionRecovery); + return configuration?.reason === PROJECT_STORAGE_ENVIRONMENT_INCOMPATIBLE && + recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false; +} + function isProjectAccessDenialBlocker(run: HeartbeatRun, options: RunFailureReportOptions): boolean { if ( run.status !== "failed" || run.errorCode !== PROJECT_ACCESS_DENIED || @@ -382,6 +396,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR isLocalPathWorkspaceConfigurationBlocker(run, options) || isRestrictedWorkspaceSourceBlocker(run, options) || isPluginRunnerAiConnectionRequiredBlocker(run, options) || + isProjectStorageEnvironmentBlocker(run, options) || isProjectAccessDenialBlocker(run, options) || isGitConnectionFailure(run, options) || isProviderQuotaFailure(run, options) ||