diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index f58ef608b00..fd665cdb6db 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1360,6 +1360,14 @@ lifecycle and ownership callbacks update its teardown state immediately, includi when preparation later throws. Public sandbox lifecycle and model fallback helpers remain re-exported by `heartbeat.ts`. +Workspace preparation is in `server/src/services/heartbeat/workspace-preparation.ts`. +`prepareHeartbeatWorkspace` resolves and provisions the workspace, persists its +metadata and issue binding, then acquires and realizes the execution environment. +It preserves the immutable workspace binding used by native recovery and records +preparation spans even when acquisition or realization fails. The executor retains +environment selection, dispatch gates, and terminal lease cleanup. Persistence +failures still clean newly created workspace artifacts before propagating. + Runtime execution is in `server/src/services/heartbeat/runtime-execution.ts`. `executeHeartbeatRuntime` dispatches a prepared native session or legacy adapter, captures usage, finalizes workspace operations, and releases gateway tokens and diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 5517583e662..a0979025f24 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -8455,12 +8455,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .set({ status: "paused" }) .where(eq(agents.id, agentId)); release(); - if (next!) - await vi.waitFor(async () => - expect((await heartbeat.getRun(next!.id))?.status).not.toBe( - "running", - ), - ); + // Wait for the released executor and its cleanup, not the default + // one-second polling window. The next case must not inherit live work. + await heartbeat.drainActiveRunExecutions(); + if (next!) expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running"); } }, ); @@ -12100,7 +12098,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const legacyReviewBefore = withLegacyReview ? await db.select().from(issues).where(eq(issues.id, legacyReviewId)) : []; - mockAdapterExecute.mockImplementationOnce(async () => { + let planOnlyRunId = ""; + mockAdapterExecute.mockImplementationOnce(async (input?: unknown) => { + planOnlyRunId = (input as { runId: string }).runId; if (withLegacyReview) { // These pre-dispatch cancellations used to satisfy both the no-comment // and churn thresholds and suppress an otherwise valid continuation. @@ -12124,17 +12124,23 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { await heartbeat.reconcileStrandedAssignedIssues(); await heartbeat.promoteDueScheduledRetries(new Date(Date.now() + 31_000)); await heartbeat.resumeQueuedRuns(); + // The continuation is produced by completion; wait for that durable boundary. + await heartbeat.drainActiveRunExecutions(); - const livenessWake = await waitForValue(async () => { + expect(planOnlyRunId).not.toBe(""); + const livenessWakes = await waitForValue(async () => { const rows = await db .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.agentId, agentId)); - return ( - rows.find((row) => row.reason === "issue_disposition_repair") ?? null - ); + // Follow-up repairs can already exist. Match this plan-only result, + // rather than relying on the database's unordered row order. + const matching = rows.filter((row) => row.reason === "issue_disposition_repair" + && row.payload?.retryOfRunId === planOnlyRunId); + return matching.length > 0 ? matching : null; }); - expect(livenessWake).toBeTruthy(); + expect(livenessWakes).toHaveLength(1); + const livenessWake = livenessWakes![0]!; expect(livenessWake?.payload).toMatchObject({ issueId, dispositionRepairAttempt: 1, @@ -12143,7 +12149,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const sourceRunId = ( livenessWake?.payload as Record | null )?.retryOfRunId; - expect(sourceRunId).toBeTruthy(); + expect(sourceRunId).toBe(planOnlyRunId); const sourceRun = await db .select() .from(heartbeatRuns) diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index ae86b02f878..576975ce900 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js"; import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js"; import { selectHeartbeatRuntime } from "./heartbeat/runtime-selection.js"; import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js"; @@ -145,19 +146,6 @@ import { readConfiguredModelFromAdapterConfig, resolveTaskSessionConfigFreshness, stripPaperclipSessionMetadataFromSessionParams, - resolveWorkspaceAfterLowTrustPreflight, - WorkspaceValidationFailure, - stripHostWorkspaceProvisionForLowTrustSandbox, - assertGitWorktreeBaseWorkspaceReady, - buildEffectiveRunWorkspaceConfigMetadata, - resolveExecutionWorkspaceConfigFreshness, - resolveExecutionWorkspaceReuseProvisioningPolicy, - provisionExecutionWorkspaceForFreshnessDecision, - mergeExecutionWorkspaceMetadataForPersistence, - resolveExecutionWorkspaceBranchOwnership, - reconcileReusedExecutionWorkspaceProjectWorkspaceId, - recordWorkspaceConfigFreshnessOperation, - prepareProjectRepositoryWorkspaces, resolveRuntimeSessionParamsForWorkspace, buildRunWorkspaceHints, buildReferencedProjectRunObservability, @@ -244,8 +232,6 @@ import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-iden import { prepareConnectionInstructionDelivery } from "./connection-instructions.js"; import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/assigned-mcp-tools.js"; -import { resolvePaperclipInstanceRoot } from "../home-paths.js"; - import { startupCancellationFence, } from "./native-runtime/native-cancellation-request.js"; @@ -341,7 +327,6 @@ import { } from "node:crypto"; import { and, - asc, desc, eq, exists, @@ -356,7 +341,6 @@ import { CHAT_PROVIDERS, ISSUE_DISPOSITION_REPAIR_RETRY_REASON, isEnvironmentDriverSupportedForAdapter, - type ExecutionWorkspace, } from "@paperclipai/shared"; import { agents, @@ -372,14 +356,9 @@ import { projectWorkspaces, } from "@paperclipai/db"; import { HttpError, notFound } from "../errors.js"; -import { - getStartupTracer, -} from "../instrumentation.js"; -import { createHostDuplexObservabilityRecorder } from "./duplex-observability-recorder.js"; -import { incrementToolRuntimeMetricCounter } from "./tool-runtime-metrics.js"; + import { logger } from "../middleware/logger.js"; import { - createGitRemoteAuthProvider, resolveManagedGitHubIdentitySelection, scrubGitCredentialText, } from "./git-credentials.js"; @@ -402,7 +381,6 @@ import { NativeCancellationPendingRecoveryError, NativeControllerDetachedForRestartError, prepareNativeWorkspaceSync, - readNativeWorkspaceSyncReference, type NativeRestartRecoveryClaim, resolveHeartbeatNativeRuntimeMode, } from "./native-runtime/index.js"; @@ -444,11 +422,10 @@ import { } from "./native-runtime/native-runner-ownership.js"; import { findNativeChatWorkspaceScope, - materializeNativeChatTaskRoot, nativeChatWorkspaceCwd, nativeChatWorkspaceMatches, } from "./native-runtime/native-chat-workspace.js"; -import { materializeIsolatedTaskDirectory, shouldUseIsolatedTaskDirectory } from "./isolated-task-directory.js"; +import { shouldUseIsolatedTaskDirectory } from "./isolated-task-directory.js"; import { trackAgentFirstHeartbeat } from "@paperclipai/shared/telemetry"; import { getTelemetryClient } from "../telemetry.js"; import { @@ -481,21 +458,13 @@ import { buildWorkspaceReadyComment, buildWorkspaceReadyMetadata, buildWorkspaceReadyPresentation, - cleanupExecutionWorkspaceArtifacts, - ensurePersistedExecutionWorkspaceAvailable, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, - realizeExecutionWorkspace, releaseRuntimeServicesForRun, - type ExecutionWorkspaceInput, type RealizedExecutionWorkspace, type RuntimeServiceRef, } from "./workspace-runtime.js"; -import { - readManagedWorktreeInstanceOwnership, - WORKTREE_INSTANCE_ROOT_METADATA_KEY, -} from "./workspace-instance-cleanup.js"; import { issueService } from "./issues.js"; import { failRunnerGoalAction, @@ -521,9 +490,7 @@ import { import { isProcessGroupAlive, } from "./local-service-supervisor.js"; -import { - isRuntimeOwnedGitBranch, -} from "./execution-workspace-branch-ownership.js"; + import { HEARTBEAT_RUN_SCRATCH_MARKER, buildHeartbeatRunScratchEnv, @@ -535,10 +502,8 @@ import { applyDefaultIsolatedExecutionWorkspacePolicy, buildExecutionWorkspaceAdapterConfig, gateProjectExecutionWorkspacePolicy, - issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, - resolveEffectiveWorkspaceStrategyType, resolveExecutionWorkspaceEnvironmentId, resolveExecutionWorkspaceMode, resolveSharedWorkspaceConcurrency, @@ -549,7 +514,6 @@ import { resolveWorktreeRunExecutionActivation, } from "./instance-settings.js"; import { - evaluateExecutionAllowlist, isExecutionForcedToKubernetes, } from "./execution-allowlist.js"; @@ -3692,862 +3656,78 @@ export function heartbeatService( if (handoff) context.paperclipFreshSessionHandoffMarkdown = handoff; } const { - selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, - workspace: resolvedWorkspace, - } = await resolveWorkspaceAfterLowTrustPreflight({ - db, - trustPreset, - isolatedWorkspacesEnabled, - effectiveExecutionWorkspaceMode, - issue: issueRef - ? { - companyId: agent.companyId, - id: issueRef.id, - projectId: issueRef.projectId, - } - : null, - resolveSelectedEnvironmentDriver: async () => { - const preflightEnvironment = await envOrchestrator.resolveEnvironment( - { - companyId: agent.companyId, - selectedEnvironmentId, - localEnvironmentId: localEnvironment.id, - }, - ); - return preflightEnvironment.driver; - }, - resolveWorkspace: async () => { - if (isDotRun) { - // This is private controller storage, never a provider filesystem. - // v6 projects workspace.access=none and cwd=null to Dot. - const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "dot-controllers", agent.companyId, run.id); - await fs.mkdir(cwd, { recursive: true, mode: 0o700 }); - return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null, - workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] }; - } - if (useIsolatedTaskDirectory && issueRef) { - const cwd = await materializeIsolatedTaskDirectory({ - companyId: agent.companyId, - issueId: issueRef.id, - }); - if (reusableExistingExecutionWorkspace && ( - reusableExistingExecutionWorkspace.companyId !== agent.companyId || - reusableExistingExecutionWorkspace.projectId !== issueRef.projectId || - reusableExistingExecutionWorkspace.sourceIssueId !== issueRef.id || - reusableExistingExecutionWorkspace.mode !== "isolated_workspace" || - reusableExistingExecutionWorkspace.strategyType !== "project_primary" || - reusableExistingExecutionWorkspace.cwd !== cwd - )) { - throw new WorkspaceValidationFailure("The existing execution workspace is not this task's isolated directory.", { - workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId: issueRef.id }, - }); - } - return resolveWorkspaceForRun(agent, context, previousSessionParams, { - executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null, - anchorWorkspace: { - cwd, - source: "task_session", - projectId: issueRef.projectId, - workspaceId: null, - repoUrl: null, - repoRef: null, - workspaceHints: [], - warnings: [], - baseCwdFallback: false, - materializationFailures: [], - }, - }); - } - if (nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId) { - const cwd = await materializeNativeChatTaskRoot( - nativeChatWorkspaceScope, - ); - return { - cwd, - source: "task_session" as const, - projectId: null, - workspaceId: null, - repoUrl: null, - repoRef: null, - workspaceHints: [], - warnings: [], - baseCwdFallback: false, - materializationFailures: [], - additionalWorkspaces: [], - referencedProjectFailures: [], - }; - } - const workspace = await resolveWorkspaceForRun( - agent, - context, - previousSessionParams, - { - useProjectWorkspace: - requestedExecutionWorkspaceMode !== "agent_default", - anchorWorkspace: requestedShouldReuseExisting && reusableExistingExecutionWorkspace?.strategyType === "git_worktree" - ? await resolveReusedGitWorkspaceAnchor({ - agent, - workspace: reusableExistingExecutionWorkspace, - responsibleUserId, - immutableNativeBinding: Boolean(nativeRecoveryExecutionWorkspaceId), - projectId: nativeRecoveryExecutionWorkspaceId - ? reusableExistingExecutionWorkspace.projectId - : issueRef?.projectId ?? readNonEmptyString(context.projectId), - explicitProjectWorkspaceId: nativeRecoveryExecutionWorkspaceId - ? reusableExistingExecutionWorkspace.projectWorkspaceId - : readNonEmptyString(context.projectWorkspaceId), - issueId, - runId: run.id, - }) - : undefined, - // Thread the selected environment driver so run-workspace resolution can tell a local - // target from a remote one, and a confined sandbox target from an unconfined remote - // target. A remote run resolves referenced projects only for the confined sandbox - // transport with the remote flag on. This never changes the anchor workspace. - executionEnvironmentDriver: - selectedEnvironmentForConfig?.driver ?? null, - }, - ); - // Additional referenced projects are a separate trusted Board - // capability, not extra readable roots for an external conversation. - return nativeChatWorkspaceScope - ? { - ...workspace, - additionalWorkspaces: [], - referencedProjectFailures: [], - } - : workspace; - }, - }); - const hostExecutionWorkspaceConfig = isDotRun ? {} : - stripHostWorkspaceProvisionForLowTrustSandbox({ - config: mergedConfig, - trustPreset, - selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, - }); - const executionWorkspaceBase = { - baseCwd: resolvedWorkspace.cwd, - source: resolvedWorkspace.source, - projectId: resolvedWorkspace.projectId, - workspaceId: resolvedWorkspace.workspaceId, - repoUrl: resolvedWorkspace.repoUrl, - repoRef: resolvedWorkspace.repoRef, - additionalWorkspaces: resolvedWorkspace.additionalWorkspaces, - } satisfies ExecutionWorkspaceInput; - await assertGitWorktreeBaseWorkspaceReady({ - requestedExecutionWorkspaceMode, - config: hostExecutionWorkspaceConfig, - issue: issueRef, - base: executionWorkspaceBase, - anchor: { - baseCwdFallback: resolvedWorkspace.baseCwdFallback, - materializationFailures: resolvedWorkspace.materializationFailures, - localPathOnlyWorkspace: resolvedWorkspace.localPathOnlyWorkspace, - }, - }); - const workspaceStrategyForFingerprint = parseObject( - hostExecutionWorkspaceConfig.workspaceStrategy, - ); - const workspaceStrategyFingerprintValue = - Object.keys(workspaceStrategyForFingerprint).length > 0 - ? workspaceStrategyForFingerprint - : null; - const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType( - requestedExecutionWorkspaceMode, + resolvedWorkspace, hostExecutionWorkspaceConfig, - ); - const selectedEnvironmentConfigForFingerprint = parseObject( - selectedEnvironmentForConfig?.config, - ); - const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig - ? { - selectionSource: environmentResolution.source, - selectedEnvironmentId, - driver: selectedEnvironmentForConfig.driver, - provider: readNonEmptyString( - selectedEnvironmentConfigForFingerprint.provider, - ), - config: selectedEnvironmentForConfig.config, - configRevisionAt: - selectedEnvironmentForConfig.updatedAt instanceof Date - ? selectedEnvironmentForConfig.updatedAt.toISOString() - : (selectedEnvironmentForConfig.updatedAt ?? null), - executionPolicy, - } - : null; - const workspaceRealizationFingerprint = { - environmentDriver: selectedEnvironmentForConfig?.driver ?? null, - environmentProvider: readNonEmptyString( - selectedEnvironmentConfigForFingerprint.provider, - ), - trustPreset: trustPreset.kind, - lowTrustSandboxDriver: lowTrustPreflightEnvironmentDriver, - }; - const workspaceFreshnessSource = resolvedWorkspace.freshnessSource ?? executionWorkspaceBase; - const latestWorkspaceConfigMetadata = - buildEffectiveRunWorkspaceConfigMetadata({ - mode: requestedExecutionWorkspaceMode, - projectId: workspaceFreshnessSource.projectId, - projectWorkspaceId: workspaceFreshnessSource.workspaceId, - strategyType: latestWorkspaceStrategyType, - workspaceStrategy: workspaceStrategyFingerprintValue, - repoUrl: workspaceFreshnessSource.repoUrl, - repoRef: - readNonEmptyString(workspaceStrategyForFingerprint.baseRef) ?? - workspaceFreshnessSource.repoRef, - configSnapshot, - environment: workspaceEnvironmentFingerprint, - realization: workspaceRealizationFingerprint, - secretManifest, - }); - const inferredExistingWorkspaceConfigMetadata = - reusableExistingExecutionWorkspace - ? buildEffectiveRunWorkspaceConfigMetadata({ - mode: issueExecutionWorkspaceModeForPersistedWorkspace( - reusableExistingExecutionWorkspace.mode, - ), - projectId: reusableExistingExecutionWorkspace.projectId, - projectWorkspaceId: - reusableExistingExecutionWorkspace.projectWorkspaceId, - strategyType: reusableExistingExecutionWorkspace.strategyType, - workspaceStrategy: workspaceStrategyFingerprintValue - ? { - ...workspaceStrategyFingerprintValue, - type: reusableExistingExecutionWorkspace.strategyType, - ...(reusableExistingExecutionWorkspace.baseRef - ? { baseRef: reusableExistingExecutionWorkspace.baseRef } - : {}), - } - : { type: reusableExistingExecutionWorkspace.strategyType }, - repoUrl: reusableExistingExecutionWorkspace.repoUrl, - repoRef: reusableExistingExecutionWorkspace.baseRef, - configSnapshot: reusableExistingExecutionWorkspace.config, - environment: workspaceEnvironmentFingerprint, - realization: workspaceRealizationFingerprint, - secretManifest, - evaluatedAt: latestWorkspaceConfigMetadata.evaluatedAt, - }) - : null; - const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness( - { - hasExistingWorkspace: - requestedShouldReuseExisting && - Boolean(reusableExistingExecutionWorkspace), - existingWorkspaceMetadata: - reusableExistingExecutionWorkspace?.metadata ?? null, - inferredMetadata: inferredExistingWorkspaceConfigMetadata, - nextMetadata: latestWorkspaceConfigMetadata, - }, - ); - const workspaceReuseProvisioningPolicy = - resolveExecutionWorkspaceReuseProvisioningPolicy({ - requestedShouldReuseExisting, - workspaceConfigFreshness, - }); - const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({ - companyId: agent.companyId, - heartbeatRunId: run.id, - executionWorkspaceId: - workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace - ? workspaceReuseRequest.requestedExecutionWorkspaceId - : null, - issueId, - }); - // The run-scoped provider resolves the active identity at each Git operation, - // including base-ref refreshes, workspace realization, and restore. - const workspaceGitAuthProvider = createGitRemoteAuthProvider( - db, - agent.companyId, - { - issueId, - heartbeatRunId: run.id, - responsibleUserId: run.responsibleUserId, - agentId: agent.id, - }, - ); - const { + latestWorkspaceConfigMetadata, + workspaceConfigFreshness, + workspaceOperationRecorder, executionWorkspace, reusedExecutionWorkspace, - policy: resolvedWorkspaceReusePolicy, - } = isDotRun ? { executionWorkspace: { ...executionWorkspaceBase, strategy: "project_primary" as const, cwd: resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: false, policy: workspaceReuseProvisioningPolicy } : await provisionExecutionWorkspaceForFreshnessDecision( - { - requestedShouldReuseExisting, - existingExecutionWorkspaceId: - workspaceReuseRequest.requestedExecutionWorkspaceId, + resolvedWorkspaceReusePolicy, + persistedExecutionWorkspace, + projectRepositoryPaths, + environmentAcquireStartedAtMs, + selectedEnvironment, + activeEnvironmentLease, + realizationResult, + environmentRealizeEndedAtMs, + workspaceRealization, + executionTarget, + workspaceRestoreSource, + } = await prepareHeartbeatWorkspace(db, { + run, agent, controllerLease, nativeRunnerPreparationSpans, + task: { issueRef, - runId: run.id, - workspaceConfigFreshness, - restoreExistingWorkspace: reusableExistingExecutionWorkspace - ? () => - ensurePersistedExecutionWorkspaceAvailable({ - db, - base: executionWorkspaceBase, - workspace: { - id: reusableExistingExecutionWorkspace.id, - mode: reusableExistingExecutionWorkspace.mode, - strategyType: - reusableExistingExecutionWorkspace.strategyType, - cwd: reusableExistingExecutionWorkspace.cwd, - providerRef: reusableExistingExecutionWorkspace.providerRef, - projectId: reusableExistingExecutionWorkspace.projectId, - projectWorkspaceId: - reusableExistingExecutionWorkspace.projectWorkspaceId, - repoUrl: reusableExistingExecutionWorkspace.repoUrl, - baseRef: reusableExistingExecutionWorkspace.baseRef, - branchName: reusableExistingExecutionWorkspace.branchName, - metadata: - reusableExistingExecutionWorkspace.metadata as Record< - string, - unknown - > | null, - config: { - provisionCommand: - configSnapshot?.provisionCommand ?? - reusableExistingExecutionWorkspace.config - ?.provisionCommand ?? - projectExecutionWorkspacePolicy?.workspaceStrategy - ?.provisionCommand ?? - null, - runtimeProvisionCommand: - configSnapshot?.runtimeProvisionCommand ?? - reusableExistingExecutionWorkspace.config - ?.runtimeProvisionCommand ?? - projectExecutionWorkspacePolicy?.workspaceStrategy - ?.runtimeProvisionCommand ?? - null, - }, - }, - issue: issueRef, - agent: { - id: agent.id, - name: agent.name, - companyId: agent.companyId, - }, - heartbeatRunId: run.id, - enableWorkspaceBranchReconcileForward: - resolvedInstanceSettings.experimental - .enableWorkspaceBranchReconcileForward, - enableWorkspaceDirtyQuarantineRepair: - resolvedInstanceSettings.experimental - .enableWorkspaceDirtyQuarantineRepair, - recorder: workspaceOperationRecorder, - resolveGitAuth: workspaceGitAuthProvider, - }) - : null, - realizeWorkspace: () => - realizeExecutionWorkspace({ - db, - base: executionWorkspaceBase, - config: hostExecutionWorkspaceConfig, - issue: issueRef, - agent: { - id: agent.id, - name: agent.name, - companyId: agent.companyId, - }, - recordedBranchOwnership: - existingExecutionWorkspace?.status !== "archived" && - existingExecutionWorkspace?.branchName - ? { - branchName: existingExecutionWorkspace.branchName, - createdByRuntime: isRuntimeOwnedGitBranch( - existingExecutionWorkspace.metadata, - ), - } - : null, - heartbeatRunId: run.id, - enableWorkspaceBranchReconcileForward: - resolvedInstanceSettings.experimental - .enableWorkspaceBranchReconcileForward, - enableWorkspaceDirtyQuarantineRepair: - resolvedInstanceSettings.experimental - .enableWorkspaceDirtyQuarantineRepair, - recorder: workspaceOperationRecorder, - resolveGitAuth: workspaceGitAuthProvider, - }), + issueId, + context, + executionProjectId, + responsibleUserId, + previousSessionParams, }, - ); - const resolvedProjectId = - executionWorkspace.projectId ?? - issueRef?.projectId ?? - executionProjectId ?? - null; - const resolvedProjectWorkspaceId = - resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace?.strategyType === "git_worktree" - ? reusableExistingExecutionWorkspace.projectWorkspaceId - : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null; - let persistedExecutionWorkspace: ExecutionWorkspace | null = null; - let issueExecutionWorkspaceIdForRun = - issueRef?.executionWorkspaceId ?? null; - let issueProjectWorkspaceIdForRun = issueRef?.projectWorkspaceId ?? null; - let issueExecutionWorkspacePreferenceForRun = - issueRef?.executionWorkspacePreference ?? null; - let issueExecutionWorkspaceModeForRun = - issueExecutionWorkspaceSettings?.mode ?? null; - const warmReusableExecutionWorkspace = - selectedEnvironmentForConfig?.driver === "sandbox" && - selectedEnvironmentConfigForFingerprint.reuseLease === true && - selectedEnvironmentConfigForFingerprint.runnerLifecycleMode === "warm"; - // Native provider checkpoints bind to the workspace row, including ordinary - // local shared workspaces. Persist that binding independently of the opt-in - // isolated-workspace UI, just as warm sandbox continuity already does. - const nativeSharedWorkspace = agent.adapterType === "paperclip_runner" && - requestedExecutionWorkspaceMode === "shared_workspace"; - const bindIssueToPersistedExecutionWorkspace = async ( - workspace: ExecutionWorkspace | null, - ) => { - if (!issueId || !workspace || nativeRecoveryExecutionWorkspaceId) { - return; - } - const nextIssueWorkspaceMode = - issueExecutionWorkspaceModeForPersistedWorkspace(workspace.mode) ?? - "agent_default"; - const shouldSwitchIssueToExistingWorkspace = - issueRef?.executionWorkspacePreference === "reuse_existing" || - requestedExecutionWorkspaceMode === "isolated_workspace" || - requestedExecutionWorkspaceMode === "operator_branch" || - warmReusableExecutionWorkspace || nativeSharedWorkspace; - const nextIssuePatch: Record = {}; - if (issueExecutionWorkspaceIdForRun !== workspace.id) { - nextIssuePatch.executionWorkspaceId = workspace.id; - } - if ( - resolvedProjectWorkspaceId && - issueProjectWorkspaceIdForRun !== resolvedProjectWorkspaceId - ) { - nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId; - } - if ( - shouldSwitchIssueToExistingWorkspace && - (issueExecutionWorkspacePreferenceForRun !== "reuse_existing" || - issueExecutionWorkspaceModeForRun !== nextIssueWorkspaceMode) - ) { - nextIssuePatch.executionWorkspacePreference = "reuse_existing"; - nextIssuePatch.executionWorkspaceSettings = { - ...(issueExecutionWorkspaceSettings ?? {}), - mode: nextIssueWorkspaceMode, - }; - } - if (Object.keys(nextIssuePatch).length > 0) { - await issuesSvc.update( - issueId, - { ...nextIssuePatch, companyGuard: agent.companyId }, - db, - undefined, - undefined, - { bindRuntimeSharedWorkspace: (warmReusableExecutionWorkspace || nativeSharedWorkspace) && workspace.mode === "shared_workspace" }, - ); - issueExecutionWorkspaceIdForRun = workspace.id; - issueProjectWorkspaceIdForRun = - resolvedProjectWorkspaceId ?? issueProjectWorkspaceIdForRun; - if (shouldSwitchIssueToExistingWorkspace) { - issueExecutionWorkspacePreferenceForRun = "reuse_existing"; - issueExecutionWorkspaceModeForRun = nextIssueWorkspaceMode; - } - } - }; - const baseExecutionWorkspaceMetadata = - mergeExecutionWorkspaceMetadataForPersistence({ - existingMetadata: - resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace - ? (reusableExistingExecutionWorkspace?.metadata ?? null) - : null, - source: executionWorkspace.source, - // Attaching a new worktree to a pre-existing branch reports a fresh - // workspace, but must not make cleanup own the operator's branch. - createdByRuntime: - resolveExecutionWorkspaceBranchOwnership(executionWorkspace), - strategyType: executionWorkspace.strategy, - configSnapshot, - shouldReuseExisting: - resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace, - shouldRefreshConfigSnapshot: - resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, - workspaceConfigMetadata: - resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata - ? latestWorkspaceConfigMetadata - : null, - baseRef: executionWorkspace.repoRef, - baseRefSha: executionWorkspace.baseRefSha ?? null, - }); - let persistedWorktreeInstanceRoot = - resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && - typeof reusableExistingExecutionWorkspace?.metadata?.[ - WORKTREE_INSTANCE_ROOT_METADATA_KEY - ] === "string" - ? reusableExistingExecutionWorkspace.metadata[ - WORKTREE_INSTANCE_ROOT_METADATA_KEY - ] - : null; - if ( - !persistedWorktreeInstanceRoot && - executionWorkspace.strategy === "git_worktree" && - executionWorkspace.worktreePath - ) { - try { - persistedWorktreeInstanceRoot = - ( - await readManagedWorktreeInstanceOwnership( - executionWorkspace.worktreePath, - ) - )?.instanceRoot ?? null; - } catch (error) { - logger.warn( - { - runId: run.id, - issueId, - executionWorkspaceCwd: executionWorkspace.cwd, - error: error instanceof Error ? error.message : String(error), - }, - "Could not record managed worktree instance ownership", - ); - } - } - const nextExecutionWorkspaceMetadata = { - ...baseExecutionWorkspaceMetadata, - ...(persistedWorktreeInstanceRoot - ? { - [WORKTREE_INSTANCE_ROOT_METADATA_KEY]: - persistedWorktreeInstanceRoot, - } - : {}), - }; - const pendingForwardBranchReconcile = - executionWorkspace.pendingForwardBranchReconcile ?? null; - const branchNameForInitialPersistence = - pendingForwardBranchReconcile?.recordedBranchName ?? - executionWorkspace.branchName; - try { - persistedExecutionWorkspace = - resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && - reusableExistingExecutionWorkspace - ? await executionWorkspacesSvc.update( - reusableExistingExecutionWorkspace.id, - { - cwd: executionWorkspace.cwd, - repoUrl: executionWorkspace.repoUrl, - baseRef: executionWorkspace.repoRef, - branchName: branchNameForInitialPersistence, - providerType: - executionWorkspace.strategy === "git_worktree" - ? "git_worktree" - : "local_fs", - providerRef: executionWorkspace.worktreePath, - status: "active", - lastUsedAt: new Date(), - metadata: nextExecutionWorkspaceMetadata, - projectWorkspaceId: - reconcileReusedExecutionWorkspaceProjectWorkspaceId( - reusableExistingExecutionWorkspace.projectWorkspaceId, - resolvedProjectWorkspaceId, - ), - }, - ) - : resolvedProjectId - ? await executionWorkspacesSvc.create({ - companyId: agent.companyId, - projectId: resolvedProjectId, - projectWorkspaceId: resolvedProjectWorkspaceId, - sourceIssueId: issueRef?.id ?? null, - mode: - requestedExecutionWorkspaceMode === "isolated_workspace" - ? "isolated_workspace" - : requestedExecutionWorkspaceMode === "operator_branch" - ? "operator_branch" - : requestedExecutionWorkspaceMode === "agent_default" - ? "adapter_managed" - : "shared_workspace", - strategyType: - executionWorkspace.strategy === "git_worktree" - ? "git_worktree" - : "project_primary", - name: - branchNameForInitialPersistence ?? - issueRef?.identifier ?? - `workspace-${agent.id.slice(0, 8)}`, - status: "active", - cwd: executionWorkspace.cwd, - repoUrl: executionWorkspace.repoUrl, - baseRef: executionWorkspace.repoRef, - branchName: branchNameForInitialPersistence, - providerType: - executionWorkspace.strategy === "git_worktree" - ? "git_worktree" - : "local_fs", - providerRef: executionWorkspace.worktreePath, - lastUsedAt: new Date(), - openedAt: new Date(), - metadata: nextExecutionWorkspaceMetadata, - }) - : null; - } catch (error) { - if (executionWorkspace.created) { - try { - await cleanupExecutionWorkspaceArtifacts({ - workspace: { - id: - reusableExistingExecutionWorkspace?.id ?? - workspaceReuseRequest.requestedExecutionWorkspaceId ?? - `transient-${run.id}`, - cwd: executionWorkspace.cwd, - providerType: - executionWorkspace.strategy === "git_worktree" - ? "git_worktree" - : "local_fs", - providerRef: executionWorkspace.worktreePath, - branchName: executionWorkspace.branchName, - repoUrl: executionWorkspace.repoUrl, - baseRef: executionWorkspace.repoRef, - projectId: resolvedProjectId, - projectWorkspaceId: resolvedProjectWorkspaceId, - sourceIssueId: issueRef?.id ?? null, - metadata: nextExecutionWorkspaceMetadata, - }, - projectWorkspace: { - cwd: resolvedWorkspace.cwd, - cleanupCommand: null, - }, - cleanupCommand: configSnapshot?.cleanupCommand ?? null, - teardownCommand: - configSnapshot?.teardownCommand ?? - projectExecutionWorkspacePolicy?.workspaceStrategy - ?.teardownCommand ?? - null, - recorder: workspaceOperationRecorder, - }); - } catch (cleanupError) { - logger.warn( - { - runId: run.id, - issueId, - executionWorkspaceCwd: executionWorkspace.cwd, - cleanupError: - cleanupError instanceof Error - ? cleanupError.message - : String(cleanupError), - }, - "Failed to cleanup realized execution workspace after persistence failure", - ); - } - } - throw error; - } - await workspaceOperationRecorder.attachExecutionWorkspaceId( - persistedExecutionWorkspace?.id ?? null, - ); - await recordWorkspaceConfigFreshnessOperation({ - recorder: workspaceOperationRecorder, - runId: run.id, - decision: workspaceConfigFreshness, - hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace), - reuseRequested: requestedShouldReuseExisting, - workspaceReused: Boolean(reusedExecutionWorkspace), - configSnapshotRefreshed: - resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, - previousWorkspaceId: - workspaceReuseRequest.requestedExecutionWorkspaceId, - activeWorkspaceId: persistedExecutionWorkspace?.id ?? null, - }); - if ( - reusableExistingExecutionWorkspace && - persistedExecutionWorkspace && - reusableExistingExecutionWorkspace.id !== - persistedExecutionWorkspace.id && - reusableExistingExecutionWorkspace.status === "active" - ) { - await executionWorkspacesSvc.update( - reusableExistingExecutionWorkspace.id, - { - status: "idle", - cleanupReason: null, - }, - ); - } - await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); - const projectRepositoryPaths: string[] = []; - if (executionWorkspace.projectId && resolvedWorkspace.source === "project_primary" && !resolvedWorkspace.baseCwdFallback) { - const repositoryRows = await db.select().from(projectWorkspaces).where(and( - eq(projectWorkspaces.companyId, agent.companyId), - eq(projectWorkspaces.projectId, executionWorkspace.projectId), - )).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id)); - const repositories = await prepareProjectRepositoryWorkspaces({ - cwd: executionWorkspace.cwd, - anchorRepoUrl: executionWorkspace.repoUrl, - workspaces: repositoryRows, - resolveGitAuth: workspaceGitAuthProvider, - }); - const paths = new Map(repositories.map((repo) => [repo.workspaceId, repo.cwd])); - projectRepositoryPaths.push(...repositories.map((repo) => path.relative(executionWorkspace.cwd, repo.cwd))); - if (resolvedWorkspace.workspaceId) paths.set(resolvedWorkspace.workspaceId, executionWorkspace.cwd); - resolvedWorkspace.workspaceHints = resolvedWorkspace.workspaceHints.map((hint) => ({ - ...hint, cwd: paths.get(hint.workspaceId) ?? hint.cwd, - })); - } - if (persistedExecutionWorkspace) { - context.executionWorkspaceId = persistedExecutionWorkspace.id; - await db - .update(heartbeatRuns) - .set({ - contextSnapshot: context, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)); - } - const environmentAcquireStartedAtMs = Date.now(); - let acquiredEnvironment: Awaited< - ReturnType - >; - try { - await controllerLease.assertOwned(); - const remoteRecovery = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner" - ? runOptions.nativeRestartRecovery : null; - const recoveryWorkspace = remoteRecovery - ? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null; - if (remoteRecovery && (!recoveryWorkspace || remoteRecovery.runId !== run.id || - recoveryWorkspace.providerLeaseId !== remoteRecovery.remote.providerLeaseId || - recoveryWorkspace.remoteCwd !== remoteRecovery.remote.remoteCwd)) { - throw new Error("native_remote_recovery_lease_mismatch"); - } - acquiredEnvironment = await envOrchestrator.acquireForRun({ - companyId: agent.companyId, - selectedEnvironmentId, - localEnvironmentId: localEnvironment.id, - adapterType: agent.adapterType, - adapterConfig: parseObject(agent.adapterConfig), - admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode, - issueId: issueId ?? null, - heartbeatRunId: run.id, - agentId: agent.id, - persistedExecutionWorkspace, - executionWorkspaceSettings: environmentExecutionWorkspaceSettings, - ...(remoteRecovery && recoveryWorkspace ? { reattachRemoteLease: { - leaseId: recoveryWorkspace.leaseId, - providerLeaseId: remoteRecovery.remote.providerLeaseId, - remoteCwd: remoteRecovery.remote.remoteCwd, - } } : {}), - }); - await controllerLease.assertOwned(); - nativeRunnerPreparationSpans.push({ - name: "environment.acquire", - parentName: "task.run", - startedAtMs: environmentAcquireStartedAtMs, - endedAtMs: Date.now(), - attributes: { adapter: agent.adapterType }, - }); - } catch (error) { - nativeRunnerPreparationSpans.push({ - name: "environment.acquire", - parentName: "task.run", - startedAtMs: environmentAcquireStartedAtMs, - endedAtMs: Date.now(), - outcome: "failed", - attributes: { adapter: agent.adapterType }, - }); - throw error; - } - const selectedEnvironment = acquiredEnvironment.environment; - // Defense-in-depth: re-check the actually-acquired environment against the - // execution allowlist. Even if selection were bypassed, a denied (local/ssh/ - // non-k8s) environment FAILS the run here rather than executing untrusted. - const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, { - driver: selectedEnvironment.driver, - provider: - typeof selectedEnvironment.config?.provider === "string" - ? selectedEnvironment.config.provider - : null, - }); - if (!allowlistDecision.allowed) { - logger.error( - { - runId: run.id, - issueId, - agentId: agent.id, - environmentId: selectedEnvironment.id, - deniedDriver: allowlistDecision.deniedDriver, - deniedProvider: allowlistDecision.deniedProvider, - }, - "Execution allowlist denied the resolved environment; failing run", - ); - throw new Error(allowlistDecision.reason); - } - let activeEnvironmentLease = { - environment: acquiredEnvironment.environment, - lease: acquiredEnvironment.lease, - leaseContext: acquiredEnvironment.leaseContext, - }; - const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder( - { - tracer: getStartupTracer(), - incrementCounter: (metric) => { - void incrementToolRuntimeMetricCounter(db, { - companyId: run.companyId, - metric, - }).catch(() => {}); - }, - emitTransportEvent: (event) => { - void (async () => { - await appendRunEvent(run, { - eventType: event.name, - stream: "system", - level: event.dimensions.outcome === "error" ? "warn" : "info", - payload: { ...event.dimensions }, - }); - })().catch(() => {}); - }, - }, - ); - const environmentRealizeStartedAtMs = Date.now(); - let realizationResult: Awaited< - ReturnType - >; - try { - realizationResult = await envOrchestrator.realizeForRun({ - environment: selectedEnvironment, - lease: activeEnvironmentLease.lease, - adapterType: agent.adapterType, - companyId: agent.companyId, - issueId: issueId ?? null, - heartbeatRunId: run.id, - executionWorkspace, + policy: { + trustPreset, + isolatedWorkspacesEnabled, effectiveExecutionWorkspaceMode, - persistedExecutionWorkspace, - duplexObservabilityRecorder, - }); - nativeRunnerPreparationSpans.push({ - name: "environment.workspace.realize", - parentName: "task.run", - startedAtMs: environmentRealizeStartedAtMs, - endedAtMs: Date.now(), - attributes: { driver: selectedEnvironment.driver }, - }); - } catch (error) { - nativeRunnerPreparationSpans.push({ - name: "environment.workspace.realize", - parentName: "task.run", - startedAtMs: environmentRealizeStartedAtMs, - endedAtMs: Date.now(), - outcome: "failed", - attributes: { driver: selectedEnvironment.driver }, - }); - throw error; - } - const environmentRealizeEndedAtMs = Date.now(); - activeEnvironmentLease = { - ...activeEnvironmentLease, - lease: realizationResult.lease, - }; - persistedExecutionWorkspace = - realizationResult.persistedExecutionWorkspace; - // A sandbox realization may materialize or replace the durable workspace - // after the host-side provisioning boundary above. Bind that final ID to - // the issue before dispatch so warm turns reuse the exact same workspace - // and lease scope instead of silently creating a per-run replacement. - await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); - const workspaceRealization = realizationResult.workspaceRealization; - const executionTarget = realizationResult.executionTarget; - // Preserve the host-owned source before adapter context can share lease - // metadata. A later copy-back failure must not adopt a rebound source. - const workspaceRestoreSource = structuredClone(realizationResult.lease); + requestedExecutionWorkspaceMode, + useIsolatedTaskDirectory, + nativeChatWorkspaceScope, + projectExecutionWorkspacePolicy, + issueExecutionWorkspaceSettings, + environmentExecutionWorkspaceSettings, + executionPolicy, + }, + environment: { + selectedEnvironmentId, + localEnvironment, + selectedEnvironmentForConfig, + environmentResolution, + resolvedInstanceSettings, + }, + config: { + mergedConfig, + configSnapshot, + secretManifest, + }, + reuse: { + requestedShouldReuseExisting, + existingExecutionWorkspace, + reusableExistingExecutionWorkspace, + workspaceReuseRequest, + nativeRecoveryExecutionWorkspaceId, + persistedNativeExecutionInput, + isDotRun, + runOptions, + }, + services: { + envOrchestrator, + executionWorkspacesSvc, + workspaceOperationsSvc, + issuesSvc, + resolveWorkspaceForRun, + resolveReusedGitWorkspaceAnchor, + appendRunEvent, + }, + }); let instructionCopy: Awaited> = null; let instructionSave: Record | null = null; const instructionPreparationKey = createHash("sha256").update(JSON.stringify({ diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts new file mode 100644 index 00000000000..c251f4e02a9 --- /dev/null +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -0,0 +1,236 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issues, projects, projectWorkspaces, type Db } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { environmentRunOrchestrator } from "../environment-run-orchestrator.js"; +import { environmentService } from "../environments.js"; +import { executionWorkspaceService } from "../execution-workspaces.js"; +import { instanceSettingsService } from "../instance-settings.js"; +import { issueService } from "../issues.js"; +import { workspaceOperationService } from "../workspace-operations.js"; +import { createHeartbeatWorkspaceResolver, resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js"; +import { prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js"; + +const execFile = promisify(execFileCallback); +const support = await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", () => { + let database: Awaited>; + let db: Db; + let home: string; + + beforeAll(async () => { + home = await mkdtemp(join(tmpdir(), "paperclip-workspace-preparation-")); + vi.stubEnv("PAPERCLIP_HOME", home); + database = await startEmbeddedPostgresTestDatabase("paperclip-workspace-preparation"); + db = createDb(database.connectionString); + await db.execute(sql`set client_min_messages = warning`); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + vi.unstubAllEnvs(); + if (home) await rm(home, { recursive: true, force: true }); + }); + beforeEach(async () => { + vi.restoreAllMocks(); + await db.execute(sql`truncate table companies restart identity cascade`); + await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true }); + }); + + async function fixture(isolated = false) { + const [company] = await db.insert(companies).values({ name: "Workspace preparation", issuePrefix: "WSP" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Workspace agent", status: "running", adapterType: "process" }).returning(); + const [project] = await db.insert(projects).values({ companyId: company.id, name: "Source project" }).returning(); + const source = join(home, project.id); + await mkdir(source); + await execFile("git", ["init", "--initial-branch=main", source]); + await execFile("git", ["-C", source, "config", "user.email", "test@example.com"]); + await execFile("git", ["-C", source, "config", "user.name", "Workspace test"]); + await writeFile(join(source, "README.md"), "source content\n"); + await execFile("git", ["-C", source, "add", "."]); + await execFile("git", ["-C", source, "commit", "-m", "Source"]); + const [projectWorkspace] = await db.insert(projectWorkspaces).values({ companyId: company.id, projectId: project.id, name: "Source", cwd: source, isPrimary: true }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Prepare this workspace", identifier: "WSP-1", projectId: project.id, projectWorkspaceId: projectWorkspace.id, assigneeAgentId: agent.id }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning(); + const localEnvironment = await environmentService(db).ensureLocalEnvironment(company.id); + const envOrchestrator = environmentRunOrchestrator(db); + const resolver = createHeartbeatWorkspaceResolver(db); + const input: HeartbeatWorkspacePreparationInput = { + run, agent, + task: { issueRef: issue, issueId: issue.id, context: { issueId: issue.id, projectId: project.id }, executionProjectId: project.id, responsibleUserId: null, previousSessionParams: null }, + policy: { + trustPreset: { kind: "standard", preset: "standard", boundary: null, sourcePresets: {} }, + isolatedWorkspacesEnabled: true, + effectiveExecutionWorkspaceMode: isolated ? "isolated_workspace" : "shared_workspace", + requestedExecutionWorkspaceMode: isolated ? "isolated_workspace" : "shared_workspace", + useIsolatedTaskDirectory: false, nativeChatWorkspaceScope: null, + projectExecutionWorkspacePolicy: null, issueExecutionWorkspaceSettings: null, environmentExecutionWorkspaceSettings: null, + executionPolicy: { executionMode: "any" }, + }, + environment: { selectedEnvironmentId: localEnvironment.id, localEnvironment, selectedEnvironmentForConfig: localEnvironment, environmentResolution: { environmentId: localEnvironment.id, source: "default" }, resolvedInstanceSettings: await instanceSettingsService(db).get() }, + config: { mergedConfig: isolated ? { workspaceStrategy: { type: "git_worktree", baseRef: "main", branchTemplate: "test-{{issue.identifier}}" } } : {}, configSnapshot: null, secretManifest: [] }, + reuse: { requestedShouldReuseExisting: false, existingExecutionWorkspace: null, reusableExistingExecutionWorkspace: null, workspaceReuseRequest: resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: null, issueExecutionWorkspacePreference: null, existingExecutionWorkspaceStatus: null }), nativeRecoveryExecutionWorkspaceId: null, persistedNativeExecutionInput: null, isDotRun: false, runOptions: {} }, + services: { envOrchestrator, executionWorkspacesSvc: executionWorkspaceService(db), workspaceOperationsSvc: workspaceOperationService(db), issuesSvc: issueService(db), ...resolver, appendRunEvent: vi.fn() }, + controllerLease: { assertOwned: vi.fn(async () => {}) }, nativeRunnerPreparationSpans: [], + }; + return { input, source, issue, run, envOrchestrator }; + } + const readIssue = (id: string) => db.select().from(issues).where(eq(issues.id, id)).then(rows => rows[0]!); + const readRun = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!); + const leases = (id: string) => db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, id)); + + async function reuse(input: HeartbeatWorkspacePreparationInput, workspace: NonNullable>["persistedExecutionWorkspace"]>) { + input.reuse.requestedShouldReuseExisting = true; + input.reuse.existingExecutionWorkspace = workspace; + input.reuse.reusableExistingExecutionWorkspace = workspace; + input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: workspace.id, issueExecutionWorkspacePreference: "reuse_existing", existingExecutionWorkspaceStatus: workspace.status }); + input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: workspace.id, executionWorkspacePreference: "reuse_existing" }; + const [run] = await db.insert(heartbeatRuns).values({ companyId: input.run.companyId, agentId: input.agent.id, issueId: input.task.issueId, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning(); + input.run = run; + } + + it("persists the workspace and issue binding before acquiring its environment", async () => { + const { input, source, issue, envOrchestrator } = await fixture(); + const acquire = envOrchestrator.acquireForRun; + vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => { + expect((await readIssue(issue.id)).executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id); + expect((await readRun(input.run.id)).contextSnapshot?.executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id); + return acquire(args); + }); + const result = await prepareHeartbeatWorkspace(db, input); + expect(result.executionWorkspace.cwd).toBe(source); + expect(result.persistedExecutionWorkspace).toMatchObject({ companyId: issue.companyId, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, mode: "shared_workspace" }); + expect(await leases(input.run.id)).toMatchObject([{ status: "active", executionWorkspaceId: result.persistedExecutionWorkspace!.id }]); + expect(input.nativeRunnerPreparationSpans.map(span => span.name)).toEqual(["environment.acquire", "environment.workspace.realize"]); + expect(input.controllerLease.assertOwned).toHaveBeenCalledTimes(2); + }); + + it("creates a real isolated worktree and reuses its recorded workspace", async () => { + const { input, source, issue } = await fixture(true); + const first = await prepareHeartbeatWorkspace(db, input); + expect(first.executionWorkspace.cwd).not.toBe(source); + expect(await readFile(join(first.executionWorkspace.cwd, "README.md"), "utf8")).toBe("source content\n"); + expect(await readIssue(issue.id)).toMatchObject({ executionWorkspaceId: first.persistedExecutionWorkspace!.id, executionWorkspacePreference: "reuse_existing" }); + await writeFile(join(first.executionWorkspace.cwd, "retained.txt"), "keep this work"); + await reuse(input, first.persistedExecutionWorkspace!); + const next = await prepareHeartbeatWorkspace(db, input); + expect(next.reusedExecutionWorkspace).toMatchObject({ cwd: first.executionWorkspace.cwd }); + expect(next.persistedExecutionWorkspace!.id).toBe(first.persistedExecutionWorkspace!.id); + expect(await readFile(join(next.executionWorkspace.cwd, "retained.txt"), "utf8")).toBe("keep this work"); + }); + + it("keeps the admitted recovery workspace when the issue now points elsewhere", async () => { + const { input, issue } = await fixture(true); + const first = await prepareHeartbeatWorkspace(db, input); + await reuse(input, first.persistedExecutionWorkspace!); + await db.update(issues).set({ executionWorkspaceId: null, projectWorkspaceId: null }).where(eq(issues.id, issue.id)); + input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: null, projectWorkspaceId: null }; + input.reuse.nativeRecoveryExecutionWorkspaceId = first.persistedExecutionWorkspace!.id; + const anchor = vi.spyOn(input.services, "resolveReusedGitWorkspaceAnchor"); + const next = await prepareHeartbeatWorkspace(db, input); + expect(next.persistedExecutionWorkspace!.id).toBe(first.persistedExecutionWorkspace!.id); + expect(anchor).toHaveBeenCalledWith(expect.objectContaining({ immutableNativeBinding: true, explicitProjectWorkspaceId: first.persistedExecutionWorkspace!.projectWorkspaceId })); + expect(await readIssue(issue.id)).toMatchObject({ executionWorkspaceId: null, projectWorkspaceId: null }); + }); + + it("removes a newly created worktree when persistence fails and preserves the original error", async () => { + const { input, source } = await fixture(true); + const error = new Error("workspace persistence unavailable"); + let createdCwd = ""; + vi.spyOn(input.services.executionWorkspacesSvc, "create").mockImplementation(async args => { createdCwd = args.cwd!; throw error; }); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error); + expect(createdCwd).not.toBe(""); + await expect(access(createdCwd)).rejects.toThrow(); + expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n"); + expect(await leases(input.run.id)).toHaveLength(0); + }); + + it("preserves a reused worktree when its metadata update fails", async () => { + const { input } = await fixture(true); + const first = await prepareHeartbeatWorkspace(db, input); + await reuse(input, first.persistedExecutionWorkspace!); + await writeFile(join(first.executionWorkspace.cwd, "retained.txt"), "unsaved work"); + const error = new Error("workspace update unavailable"); + vi.spyOn(input.services.executionWorkspacesSvc, "update").mockRejectedValueOnce(error); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error); + expect(await readFile(join(first.executionWorkspace.cwd, "retained.txt"), "utf8")).toBe("unsaved work"); + expect(await leases(input.run.id)).toHaveLength(0); + }); + + it.each(["acquire", "realize", "ownership"] as const)("preserves %s errors and durable lease state for outer cleanup", async failure => { + const { input, envOrchestrator } = await fixture(); + const error = new Error(`${failure} failed`); + if (failure === "acquire") vi.spyOn(envOrchestrator, "acquireForRun").mockRejectedValueOnce(error); + if (failure === "realize") vi.spyOn(envOrchestrator, "realizeForRun").mockRejectedValueOnce(error); + if (failure === "ownership") vi.mocked(input.controllerLease.assertOwned).mockResolvedValueOnce(undefined).mockRejectedValueOnce(error); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error); + expect(input.nativeRunnerPreparationSpans.at(-1)).toMatchObject({ name: failure === "realize" ? "environment.workspace.realize" : "environment.acquire", outcome: "failed" }); + expect((await readRun(input.run.id)).status).toBe("running"); + expect(await leases(input.run.id)).toHaveLength(failure === "acquire" ? 0 : 1); + if (failure !== "acquire") { + expect(await leases(input.run.id)).toMatchObject([{ status: "active" }]); + // The production executor terminalizes first, then releases by durable run ID. + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, input.run.id)); + await envOrchestrator.releaseForRun({ heartbeatRunId: input.run.id, companyId: input.run.companyId, agentId: input.agent.id, status: "released" }); + expect(await leases(input.run.id)).toMatchObject([{ status: "released" }]); + } + }); + + it("checks the acquired environment allowlist before realization", async () => { + const { input, envOrchestrator } = await fixture(); + input.policy.executionPolicy = { managedSandboxOnly: true }; + const realize = vi.spyOn(envOrchestrator, "realizeForRun"); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("forbids local execution"); + expect(realize).not.toHaveBeenCalled(); + expect(await leases(input.run.id)).toMatchObject([{ status: "active" }]); + }); + + it("rejects low-trust local execution before resolving or provisioning a workspace", async () => { + const { input } = await fixture(true); + input.policy.trustPreset = { + kind: "low_trust_review", preset: "low_trust_review", sourcePresets: {}, + boundary: { mode: "low_trust_review", companyId: input.agent.companyId, rootIssueId: input.task.issueId! }, + }; + const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun"); + const create = vi.spyOn(input.services.executionWorkspacesSvc, "create"); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("requires a sandbox environment"); + expect(resolve).not.toHaveBeenCalled(); + expect(create).not.toHaveBeenCalled(); + expect(await leases(input.run.id)).toHaveLength(0); + }); + + it("rejects remote recovery without its retained sync binding before acquiring a lease", async () => { + const { input, envOrchestrator } = await fixture(); + input.reuse.runOptions.nativeRestartRecovery = { + kind: "reattach_remote_runner", runId: input.run.id, leaseOwner: "controller", + controllerGeneration: 1, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null, + remote: { providerLeaseId: "original-provider-lease", remoteCwd: "/original" }, + }; + const acquire = vi.spyOn(envOrchestrator, "acquireForRun"); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("native_remote_recovery_lease_mismatch"); + expect(acquire).not.toHaveBeenCalled(); + expect(await leases(input.run.id)).toHaveLength(0); + expect(input.nativeRunnerPreparationSpans).toMatchObject([{ name: "environment.acquire", outcome: "failed" }]); + }); + + it("binds a replacement workspace from realization and snapshots the restore lease", async () => { + const { input, envOrchestrator, issue } = await fixture(); + const realize = envOrchestrator.realizeForRun; + vi.spyOn(envOrchestrator, "realizeForRun").mockImplementation(async args => { + const result = await realize(args); + const workspace = await input.services.executionWorkspacesSvc.create({ companyId: issue.companyId, projectId: issue.projectId!, projectWorkspaceId: issue.projectWorkspaceId, sourceIssueId: issue.id, name: "Realized workspace", mode: "shared_workspace", strategyType: "project_primary", cwd: args.executionWorkspace.cwd }); + return { ...result, persistedExecutionWorkspace: workspace, lease: { ...result.lease, metadata: { remoteCwd: "/original" } } }; + }); + const result = await prepareHeartbeatWorkspace(db, input); + expect((await readIssue(issue.id)).executionWorkspaceId).toBe(result.persistedExecutionWorkspace!.id); + expect(result.persistedExecutionWorkspace!.name).toBe("Realized workspace"); + result.activeEnvironmentLease.lease.metadata!.remoteCwd = "/rebound"; + expect(result.workspaceRestoreSource.metadata?.remoteCwd).toBe("/original"); + }); +}); diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts new file mode 100644 index 00000000000..cb699b94820 --- /dev/null +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -0,0 +1,1072 @@ +import { + resolveWorkspaceAfterLowTrustPreflight, + WorkspaceValidationFailure, + stripHostWorkspaceProvisionForLowTrustSandbox, + assertGitWorktreeBaseWorkspaceReady, + buildEffectiveRunWorkspaceConfigMetadata, + resolveExecutionWorkspaceConfigFreshness, + resolveExecutionWorkspaceReuseProvisioningPolicy, + provisionExecutionWorkspaceForFreshnessDecision, + mergeExecutionWorkspaceMetadataForPersistence, + resolveExecutionWorkspaceBranchOwnership, + reconcileReusedExecutionWorkspaceProjectWorkspaceId, + recordWorkspaceConfigFreshnessOperation, + prepareProjectRepositoryWorkspaces, +} from "./workspaces.js"; +import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { and, asc, eq } from "drizzle-orm"; +import { type ExecutionWorkspace } from "@paperclipai/shared"; +import { heartbeatRuns, projectWorkspaces } from "@paperclipai/db"; +import { getStartupTracer } from "../../instrumentation.js"; +import { createHostDuplexObservabilityRecorder } from "../duplex-observability-recorder.js"; +import { incrementToolRuntimeMetricCounter } from "../tool-runtime-metrics.js"; +import { logger } from "../../middleware/logger.js"; +import { createGitRemoteAuthProvider } from "../git-credentials.js"; +import { readNativeWorkspaceSyncReference } from "../native-runtime/index.js"; +import { parseObject } from "../../adapters/utils.js"; +import { materializeNativeChatTaskRoot } from "../native-runtime/native-chat-workspace.js"; +import { materializeIsolatedTaskDirectory } from "../isolated-task-directory.js"; +import { + cleanupExecutionWorkspaceArtifacts, + ensurePersistedExecutionWorkspaceAvailable, + realizeExecutionWorkspace, + type ExecutionWorkspaceInput, + type RealizedExecutionWorkspace, +} from "../workspace-runtime.js"; +import { + readManagedWorktreeInstanceOwnership, + WORKTREE_INSTANCE_ROOT_METADATA_KEY, +} from "../workspace-instance-cleanup.js"; +import { isRuntimeOwnedGitBranch } from "../execution-workspace-branch-ownership.js"; +import { + issueExecutionWorkspaceModeForPersistedWorkspace, + resolveEffectiveWorkspaceStrategyType, +} from "../execution-workspace-policy.js"; +import { evaluateExecutionAllowlist } from "../execution-allowlist.js"; +import type { Db, agents } from "@paperclipai/db"; +import type { + Environment, + IssueExecutionWorkspaceSettings, + ProjectExecutionWorkspacePolicy, +} from "@paperclipai/shared"; +import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import type { NativeRunHistoricalSpan } from "../native-runtime/native-run-trace.js"; +import type { NativeRestartRecoveryClaim } from "../native-runtime/index.js"; +import type { NativeChatWorkspaceScope } from "../native-runtime/native-chat-workspace.js"; +import type { TrustPresetResolution } from "../trust-preset-resolver.js"; +import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js"; +import type { instanceSettingsService } from "../instance-settings.js"; +import type { issueService } from "../issues.js"; +import type { executionWorkspaceService } from "../execution-workspaces.js"; +import type { workspaceOperationService } from "../workspace-operations.js"; +import type { watchLegacyControllerLease } from "../legacy-controller-lease.js"; +import type { + createHeartbeatWorkspaceResolver, + resolveExecutionWorkspaceReuseRequestForIssue, +} from "./workspaces.js"; +import type { createHeartbeatLifecycle } from "./run-lifecycle.js"; +import type { + ParsedExecutionWorkspaceMode, + ExecutionWorkspaceEnvironmentResolution, +} from "../execution-workspace-policy.js"; + +type WorkspaceResolver = ReturnType; +type WorkspaceMetadataInput = Parameters[0]; + +/** Prepares the workspace and environment; dispatch and terminal lease release stay with the caller. */ +export interface HeartbeatWorkspacePreparationInput { + run: typeof heartbeatRuns.$inferSelect; + agent: typeof agents.$inferSelect; + task: { + issueRef: (NonNullable[0]["issue"]> & { + projectId: string | null; + projectWorkspaceId: string | null; + executionWorkspaceId: string | null; + executionWorkspacePreference: string | null; + }) | null; + issueId: string | null; + context: Record; + executionProjectId: string | null; + responsibleUserId: string | null; + previousSessionParams: Record | null; + }; + policy: { + trustPreset: TrustPresetResolution; + isolatedWorkspacesEnabled: boolean; + effectiveExecutionWorkspaceMode: ParsedExecutionWorkspaceMode; + requestedExecutionWorkspaceMode: ParsedExecutionWorkspaceMode; + useIsolatedTaskDirectory: boolean; + nativeChatWorkspaceScope: NativeChatWorkspaceScope | null; + projectExecutionWorkspacePolicy: ProjectExecutionWorkspacePolicy | null; + issueExecutionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; + environmentExecutionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; + executionPolicy: Parameters[0]; + }; + environment: { + selectedEnvironmentId: string; + localEnvironment: Environment; + selectedEnvironmentForConfig: Environment | null; + environmentResolution: ExecutionWorkspaceEnvironmentResolution; + resolvedInstanceSettings: Awaited["get"]>>; + }; + config: { + mergedConfig: Record; + configSnapshot: WorkspaceMetadataInput["configSnapshot"]; + secretManifest: WorkspaceMetadataInput["secretManifest"]; + }; + reuse: { + requestedShouldReuseExisting: boolean; + existingExecutionWorkspace: ExecutionWorkspace | null; + reusableExistingExecutionWorkspace: ExecutionWorkspace | null; + workspaceReuseRequest: ReturnType; + nativeRecoveryExecutionWorkspaceId: string | null; + persistedNativeExecutionInput: NativeExecutionInput | null; + isDotRun: boolean; + runOptions: { nativeRestartRecovery?: NativeRestartRecoveryClaim }; + }; + services: { + envOrchestrator: Pick, "resolveEnvironment" | "acquireForRun" | "realizeForRun">; + executionWorkspacesSvc: Pick, "create" | "update">; + workspaceOperationsSvc: Pick, "createRecorder">; + issuesSvc: Pick, "update">; + resolveWorkspaceForRun: WorkspaceResolver["resolveWorkspaceForRun"]; + resolveReusedGitWorkspaceAnchor: WorkspaceResolver["resolveReusedGitWorkspaceAnchor"]; + appendRunEvent: ReturnType["appendRunEvent"]; + }; + controllerLease: Pick, "assertOwned">; + nativeRunnerPreparationSpans: NativeRunHistoricalSpan[]; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +export async function prepareHeartbeatWorkspace(db: Db, input: HeartbeatWorkspacePreparationInput) { + const { run, agent, controllerLease, nativeRunnerPreparationSpans } = input; + const { + issueRef, + issueId, + context, + executionProjectId, + responsibleUserId, + previousSessionParams, + } = input.task; + const { + trustPreset, + isolatedWorkspacesEnabled, + effectiveExecutionWorkspaceMode, + requestedExecutionWorkspaceMode, + useIsolatedTaskDirectory, + nativeChatWorkspaceScope, + projectExecutionWorkspacePolicy, + issueExecutionWorkspaceSettings, + environmentExecutionWorkspaceSettings, + executionPolicy, + } = input.policy; + const { + selectedEnvironmentId, + localEnvironment, + selectedEnvironmentForConfig, + environmentResolution, + resolvedInstanceSettings, + } = input.environment; + const { mergedConfig, configSnapshot, secretManifest } = input.config; + const { + requestedShouldReuseExisting, + existingExecutionWorkspace, + reusableExistingExecutionWorkspace, + workspaceReuseRequest, + nativeRecoveryExecutionWorkspaceId, + persistedNativeExecutionInput, + isDotRun, + runOptions, + } = input.reuse; + const { + envOrchestrator, + executionWorkspacesSvc, + workspaceOperationsSvc, + issuesSvc, + resolveWorkspaceForRun, + resolveReusedGitWorkspaceAnchor, + appendRunEvent, + } = input.services; + const { + selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, + workspace: resolvedWorkspace, + } = await resolveWorkspaceAfterLowTrustPreflight({ + db, + trustPreset, + isolatedWorkspacesEnabled, + effectiveExecutionWorkspaceMode, + issue: issueRef + ? { + companyId: agent.companyId, + id: issueRef.id, + projectId: issueRef.projectId, + } + : null, + resolveSelectedEnvironmentDriver: async () => { + const preflightEnvironment = await envOrchestrator.resolveEnvironment( + { + companyId: agent.companyId, + selectedEnvironmentId, + localEnvironmentId: localEnvironment.id, + }, + ); + return preflightEnvironment.driver; + }, + resolveWorkspace: async () => { + if (isDotRun) { + // This is private controller storage, never a provider filesystem. + // v6 projects workspace.access=none and cwd=null to Dot. + const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "dot-controllers", agent.companyId, run.id); + await fs.mkdir(cwd, { recursive: true, mode: 0o700 }); + return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null, + workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] }; + } + if (useIsolatedTaskDirectory && issueRef) { + const cwd = await materializeIsolatedTaskDirectory({ + companyId: agent.companyId, + issueId: issueRef.id, + }); + if (reusableExistingExecutionWorkspace && ( + reusableExistingExecutionWorkspace.companyId !== agent.companyId || + reusableExistingExecutionWorkspace.projectId !== issueRef.projectId || + reusableExistingExecutionWorkspace.sourceIssueId !== issueRef.id || + reusableExistingExecutionWorkspace.mode !== "isolated_workspace" || + reusableExistingExecutionWorkspace.strategyType !== "project_primary" || + reusableExistingExecutionWorkspace.cwd !== cwd + )) { + throw new WorkspaceValidationFailure("The existing execution workspace is not this task's isolated directory.", { + workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId: issueRef.id }, + }); + } + return resolveWorkspaceForRun(agent, context, previousSessionParams, { + executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null, + anchorWorkspace: { + cwd, + source: "task_session", + projectId: issueRef.projectId, + workspaceId: null, + repoUrl: null, + repoRef: null, + workspaceHints: [], + warnings: [], + baseCwdFallback: false, + materializationFailures: [], + }, + }); + } + if (nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId) { + const cwd = await materializeNativeChatTaskRoot( + nativeChatWorkspaceScope, + ); + return { + cwd, + source: "task_session" as const, + projectId: null, + workspaceId: null, + repoUrl: null, + repoRef: null, + workspaceHints: [], + warnings: [], + baseCwdFallback: false, + materializationFailures: [], + additionalWorkspaces: [], + referencedProjectFailures: [], + }; + } + const workspace = await resolveWorkspaceForRun( + agent, + context, + previousSessionParams, + { + useProjectWorkspace: + requestedExecutionWorkspaceMode !== "agent_default", + anchorWorkspace: requestedShouldReuseExisting && reusableExistingExecutionWorkspace?.strategyType === "git_worktree" + ? await resolveReusedGitWorkspaceAnchor({ + agent, + workspace: reusableExistingExecutionWorkspace, + responsibleUserId, + immutableNativeBinding: Boolean(nativeRecoveryExecutionWorkspaceId), + projectId: nativeRecoveryExecutionWorkspaceId + ? reusableExistingExecutionWorkspace.projectId + : issueRef?.projectId ?? readNonEmptyString(context.projectId), + explicitProjectWorkspaceId: nativeRecoveryExecutionWorkspaceId + ? reusableExistingExecutionWorkspace.projectWorkspaceId + : readNonEmptyString(context.projectWorkspaceId), + issueId, + runId: run.id, + }) + : undefined, + // Thread the selected environment driver so run-workspace resolution can tell a local + // target from a remote one, and a confined sandbox target from an unconfined remote + // target. A remote run resolves referenced projects only for the confined sandbox + // transport with the remote flag on. This never changes the anchor workspace. + executionEnvironmentDriver: + selectedEnvironmentForConfig?.driver ?? null, + }, + ); + // Additional referenced projects are a separate trusted Board + // capability, not extra readable roots for an external conversation. + return nativeChatWorkspaceScope + ? { + ...workspace, + additionalWorkspaces: [], + referencedProjectFailures: [], + } + : workspace; + }, + }); + const hostExecutionWorkspaceConfig = isDotRun ? {} : + stripHostWorkspaceProvisionForLowTrustSandbox({ + config: mergedConfig, + trustPreset, + selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, + }); + const executionWorkspaceBase = { + baseCwd: resolvedWorkspace.cwd, + source: resolvedWorkspace.source, + projectId: resolvedWorkspace.projectId, + workspaceId: resolvedWorkspace.workspaceId, + repoUrl: resolvedWorkspace.repoUrl, + repoRef: resolvedWorkspace.repoRef, + additionalWorkspaces: resolvedWorkspace.additionalWorkspaces, + } satisfies ExecutionWorkspaceInput; + await assertGitWorktreeBaseWorkspaceReady({ + requestedExecutionWorkspaceMode, + config: hostExecutionWorkspaceConfig, + issue: issueRef, + base: executionWorkspaceBase, + anchor: { + baseCwdFallback: resolvedWorkspace.baseCwdFallback, + materializationFailures: resolvedWorkspace.materializationFailures, + localPathOnlyWorkspace: resolvedWorkspace.localPathOnlyWorkspace, + }, + }); + const workspaceStrategyForFingerprint = parseObject( + hostExecutionWorkspaceConfig.workspaceStrategy, + ); + const workspaceStrategyFingerprintValue = + Object.keys(workspaceStrategyForFingerprint).length > 0 + ? workspaceStrategyForFingerprint + : null; + const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType( + requestedExecutionWorkspaceMode, + hostExecutionWorkspaceConfig, + ); + const selectedEnvironmentConfigForFingerprint = parseObject( + selectedEnvironmentForConfig?.config, + ); + const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig + ? { + selectionSource: environmentResolution.source, + selectedEnvironmentId, + driver: selectedEnvironmentForConfig.driver, + provider: readNonEmptyString( + selectedEnvironmentConfigForFingerprint.provider, + ), + config: selectedEnvironmentForConfig.config, + configRevisionAt: + selectedEnvironmentForConfig.updatedAt instanceof Date + ? selectedEnvironmentForConfig.updatedAt.toISOString() + : (selectedEnvironmentForConfig.updatedAt ?? null), + executionPolicy, + } + : null; + const workspaceRealizationFingerprint = { + environmentDriver: selectedEnvironmentForConfig?.driver ?? null, + environmentProvider: readNonEmptyString( + selectedEnvironmentConfigForFingerprint.provider, + ), + trustPreset: trustPreset.kind, + lowTrustSandboxDriver: lowTrustPreflightEnvironmentDriver, + }; + const workspaceFreshnessSource = resolvedWorkspace.freshnessSource ?? executionWorkspaceBase; + const latestWorkspaceConfigMetadata = + buildEffectiveRunWorkspaceConfigMetadata({ + mode: requestedExecutionWorkspaceMode, + projectId: workspaceFreshnessSource.projectId, + projectWorkspaceId: workspaceFreshnessSource.workspaceId, + strategyType: latestWorkspaceStrategyType, + workspaceStrategy: workspaceStrategyFingerprintValue, + repoUrl: workspaceFreshnessSource.repoUrl, + repoRef: + readNonEmptyString(workspaceStrategyForFingerprint.baseRef) ?? + workspaceFreshnessSource.repoRef, + configSnapshot, + environment: workspaceEnvironmentFingerprint, + realization: workspaceRealizationFingerprint, + secretManifest, + }); + const inferredExistingWorkspaceConfigMetadata = + reusableExistingExecutionWorkspace + ? buildEffectiveRunWorkspaceConfigMetadata({ + mode: issueExecutionWorkspaceModeForPersistedWorkspace( + reusableExistingExecutionWorkspace.mode, + ), + projectId: reusableExistingExecutionWorkspace.projectId, + projectWorkspaceId: + reusableExistingExecutionWorkspace.projectWorkspaceId, + strategyType: reusableExistingExecutionWorkspace.strategyType, + workspaceStrategy: workspaceStrategyFingerprintValue + ? { + ...workspaceStrategyFingerprintValue, + type: reusableExistingExecutionWorkspace.strategyType, + ...(reusableExistingExecutionWorkspace.baseRef + ? { baseRef: reusableExistingExecutionWorkspace.baseRef } + : {}), + } + : { type: reusableExistingExecutionWorkspace.strategyType }, + repoUrl: reusableExistingExecutionWorkspace.repoUrl, + repoRef: reusableExistingExecutionWorkspace.baseRef, + configSnapshot: reusableExistingExecutionWorkspace.config, + environment: workspaceEnvironmentFingerprint, + realization: workspaceRealizationFingerprint, + secretManifest, + evaluatedAt: latestWorkspaceConfigMetadata.evaluatedAt, + }) + : null; + const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness( + { + hasExistingWorkspace: + requestedShouldReuseExisting && + Boolean(reusableExistingExecutionWorkspace), + existingWorkspaceMetadata: + reusableExistingExecutionWorkspace?.metadata ?? null, + inferredMetadata: inferredExistingWorkspaceConfigMetadata, + nextMetadata: latestWorkspaceConfigMetadata, + }, + ); + const workspaceReuseProvisioningPolicy = + resolveExecutionWorkspaceReuseProvisioningPolicy({ + requestedShouldReuseExisting, + workspaceConfigFreshness, + }); + const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({ + companyId: agent.companyId, + heartbeatRunId: run.id, + executionWorkspaceId: + workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace + ? workspaceReuseRequest.requestedExecutionWorkspaceId + : null, + issueId, + }); + // The run-scoped provider resolves the active identity at each Git operation, + // including base-ref refreshes, workspace realization, and restore. + const workspaceGitAuthProvider = createGitRemoteAuthProvider( + db, + agent.companyId, + { + issueId, + heartbeatRunId: run.id, + responsibleUserId: run.responsibleUserId, + agentId: agent.id, + }, + ); + const { + executionWorkspace, + reusedExecutionWorkspace, + policy: resolvedWorkspaceReusePolicy, + } = isDotRun ? { executionWorkspace: { ...executionWorkspaceBase, strategy: "project_primary" as const, cwd: resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: false, policy: workspaceReuseProvisioningPolicy } : await provisionExecutionWorkspaceForFreshnessDecision( + { + requestedShouldReuseExisting, + existingExecutionWorkspaceId: + workspaceReuseRequest.requestedExecutionWorkspaceId, + issueRef, + runId: run.id, + workspaceConfigFreshness, + restoreExistingWorkspace: reusableExistingExecutionWorkspace + ? () => + ensurePersistedExecutionWorkspaceAvailable({ + db, + base: executionWorkspaceBase, + workspace: { + id: reusableExistingExecutionWorkspace.id, + mode: reusableExistingExecutionWorkspace.mode, + strategyType: + reusableExistingExecutionWorkspace.strategyType, + cwd: reusableExistingExecutionWorkspace.cwd, + providerRef: reusableExistingExecutionWorkspace.providerRef, + projectId: reusableExistingExecutionWorkspace.projectId, + projectWorkspaceId: + reusableExistingExecutionWorkspace.projectWorkspaceId, + repoUrl: reusableExistingExecutionWorkspace.repoUrl, + baseRef: reusableExistingExecutionWorkspace.baseRef, + branchName: reusableExistingExecutionWorkspace.branchName, + metadata: + reusableExistingExecutionWorkspace.metadata as Record< + string, + unknown + > | null, + config: { + provisionCommand: + configSnapshot?.provisionCommand ?? + reusableExistingExecutionWorkspace.config + ?.provisionCommand ?? + projectExecutionWorkspacePolicy?.workspaceStrategy + ?.provisionCommand ?? + null, + runtimeProvisionCommand: + configSnapshot?.runtimeProvisionCommand ?? + reusableExistingExecutionWorkspace.config + ?.runtimeProvisionCommand ?? + projectExecutionWorkspacePolicy?.workspaceStrategy + ?.runtimeProvisionCommand ?? + null, + }, + }, + issue: issueRef, + agent: { + id: agent.id, + name: agent.name, + companyId: agent.companyId, + }, + heartbeatRunId: run.id, + enableWorkspaceBranchReconcileForward: + resolvedInstanceSettings.experimental + .enableWorkspaceBranchReconcileForward, + enableWorkspaceDirtyQuarantineRepair: + resolvedInstanceSettings.experimental + .enableWorkspaceDirtyQuarantineRepair, + recorder: workspaceOperationRecorder, + resolveGitAuth: workspaceGitAuthProvider, + }) + : null, + realizeWorkspace: () => + realizeExecutionWorkspace({ + db, + base: executionWorkspaceBase, + config: hostExecutionWorkspaceConfig, + issue: issueRef, + agent: { + id: agent.id, + name: agent.name, + companyId: agent.companyId, + }, + recordedBranchOwnership: + existingExecutionWorkspace?.status !== "archived" && + existingExecutionWorkspace?.branchName + ? { + branchName: existingExecutionWorkspace.branchName, + createdByRuntime: isRuntimeOwnedGitBranch( + existingExecutionWorkspace.metadata, + ), + } + : null, + heartbeatRunId: run.id, + enableWorkspaceBranchReconcileForward: + resolvedInstanceSettings.experimental + .enableWorkspaceBranchReconcileForward, + enableWorkspaceDirtyQuarantineRepair: + resolvedInstanceSettings.experimental + .enableWorkspaceDirtyQuarantineRepair, + recorder: workspaceOperationRecorder, + resolveGitAuth: workspaceGitAuthProvider, + }), + }, + ); + const resolvedProjectId = + executionWorkspace.projectId ?? + issueRef?.projectId ?? + executionProjectId ?? + null; + const resolvedProjectWorkspaceId = + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace?.strategyType === "git_worktree" + ? reusableExistingExecutionWorkspace.projectWorkspaceId + : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null; + let persistedExecutionWorkspace: ExecutionWorkspace | null = null; + let issueExecutionWorkspaceIdForRun = + issueRef?.executionWorkspaceId ?? null; + let issueProjectWorkspaceIdForRun = issueRef?.projectWorkspaceId ?? null; + let issueExecutionWorkspacePreferenceForRun = + issueRef?.executionWorkspacePreference ?? null; + let issueExecutionWorkspaceModeForRun = + issueExecutionWorkspaceSettings?.mode ?? null; + const warmReusableExecutionWorkspace = + selectedEnvironmentForConfig?.driver === "sandbox" && + selectedEnvironmentConfigForFingerprint.reuseLease === true && + selectedEnvironmentConfigForFingerprint.runnerLifecycleMode === "warm"; + // Native provider checkpoints bind to the workspace row, including ordinary + // local shared workspaces. Persist that binding independently of the opt-in + // isolated-workspace UI, just as warm sandbox continuity already does. + const nativeSharedWorkspace = agent.adapterType === "paperclip_runner" && + requestedExecutionWorkspaceMode === "shared_workspace"; + const bindIssueToPersistedExecutionWorkspace = async ( + workspace: ExecutionWorkspace | null, + ) => { + if (!issueId || !workspace || nativeRecoveryExecutionWorkspaceId) { + return; + } + const nextIssueWorkspaceMode = + issueExecutionWorkspaceModeForPersistedWorkspace(workspace.mode) ?? + "agent_default"; + const shouldSwitchIssueToExistingWorkspace = + issueRef?.executionWorkspacePreference === "reuse_existing" || + requestedExecutionWorkspaceMode === "isolated_workspace" || + requestedExecutionWorkspaceMode === "operator_branch" || + warmReusableExecutionWorkspace || nativeSharedWorkspace; + const nextIssuePatch: Record = {}; + if (issueExecutionWorkspaceIdForRun !== workspace.id) { + nextIssuePatch.executionWorkspaceId = workspace.id; + } + if ( + resolvedProjectWorkspaceId && + issueProjectWorkspaceIdForRun !== resolvedProjectWorkspaceId + ) { + nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId; + } + if ( + shouldSwitchIssueToExistingWorkspace && + (issueExecutionWorkspacePreferenceForRun !== "reuse_existing" || + issueExecutionWorkspaceModeForRun !== nextIssueWorkspaceMode) + ) { + nextIssuePatch.executionWorkspacePreference = "reuse_existing"; + nextIssuePatch.executionWorkspaceSettings = { + ...(issueExecutionWorkspaceSettings ?? {}), + mode: nextIssueWorkspaceMode, + }; + } + if (Object.keys(nextIssuePatch).length > 0) { + await issuesSvc.update( + issueId, + { ...nextIssuePatch, companyGuard: agent.companyId }, + db, + undefined, + undefined, + { bindRuntimeSharedWorkspace: (warmReusableExecutionWorkspace || nativeSharedWorkspace) && workspace.mode === "shared_workspace" }, + ); + issueExecutionWorkspaceIdForRun = workspace.id; + issueProjectWorkspaceIdForRun = + resolvedProjectWorkspaceId ?? issueProjectWorkspaceIdForRun; + if (shouldSwitchIssueToExistingWorkspace) { + issueExecutionWorkspacePreferenceForRun = "reuse_existing"; + issueExecutionWorkspaceModeForRun = nextIssueWorkspaceMode; + } + } + }; + const baseExecutionWorkspaceMetadata = + mergeExecutionWorkspaceMetadataForPersistence({ + existingMetadata: + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace + ? (reusableExistingExecutionWorkspace?.metadata ?? null) + : null, + source: executionWorkspace.source, + // Attaching a new worktree to a pre-existing branch reports a fresh + // workspace, but must not make cleanup own the operator's branch. + createdByRuntime: + resolveExecutionWorkspaceBranchOwnership(executionWorkspace), + strategyType: executionWorkspace.strategy, + configSnapshot, + shouldReuseExisting: + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace, + shouldRefreshConfigSnapshot: + resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, + workspaceConfigMetadata: + resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata + ? latestWorkspaceConfigMetadata + : null, + baseRef: executionWorkspace.repoRef, + baseRefSha: executionWorkspace.baseRefSha ?? null, + }); + let persistedWorktreeInstanceRoot = + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && + typeof reusableExistingExecutionWorkspace?.metadata?.[ + WORKTREE_INSTANCE_ROOT_METADATA_KEY + ] === "string" + ? reusableExistingExecutionWorkspace.metadata[ + WORKTREE_INSTANCE_ROOT_METADATA_KEY + ] + : null; + if ( + !persistedWorktreeInstanceRoot && + executionWorkspace.strategy === "git_worktree" && + executionWorkspace.worktreePath + ) { + try { + persistedWorktreeInstanceRoot = + ( + await readManagedWorktreeInstanceOwnership( + executionWorkspace.worktreePath, + ) + )?.instanceRoot ?? null; + } catch (error) { + logger.warn( + { + runId: run.id, + issueId, + executionWorkspaceCwd: executionWorkspace.cwd, + error: error instanceof Error ? error.message : String(error), + }, + "Could not record managed worktree instance ownership", + ); + } + } + const nextExecutionWorkspaceMetadata = { + ...baseExecutionWorkspaceMetadata, + ...(persistedWorktreeInstanceRoot + ? { + [WORKTREE_INSTANCE_ROOT_METADATA_KEY]: + persistedWorktreeInstanceRoot, + } + : {}), + }; + const pendingForwardBranchReconcile = + executionWorkspace.pendingForwardBranchReconcile ?? null; + const branchNameForInitialPersistence = + pendingForwardBranchReconcile?.recordedBranchName ?? + executionWorkspace.branchName; + try { + persistedExecutionWorkspace = + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && + reusableExistingExecutionWorkspace + ? await executionWorkspacesSvc.update( + reusableExistingExecutionWorkspace.id, + { + cwd: executionWorkspace.cwd, + repoUrl: executionWorkspace.repoUrl, + baseRef: executionWorkspace.repoRef, + branchName: branchNameForInitialPersistence, + providerType: + executionWorkspace.strategy === "git_worktree" + ? "git_worktree" + : "local_fs", + providerRef: executionWorkspace.worktreePath, + status: "active", + lastUsedAt: new Date(), + metadata: nextExecutionWorkspaceMetadata, + projectWorkspaceId: + reconcileReusedExecutionWorkspaceProjectWorkspaceId( + reusableExistingExecutionWorkspace.projectWorkspaceId, + resolvedProjectWorkspaceId, + ), + }, + ) + : resolvedProjectId + ? await executionWorkspacesSvc.create({ + companyId: agent.companyId, + projectId: resolvedProjectId, + projectWorkspaceId: resolvedProjectWorkspaceId, + sourceIssueId: issueRef?.id ?? null, + mode: + requestedExecutionWorkspaceMode === "isolated_workspace" + ? "isolated_workspace" + : requestedExecutionWorkspaceMode === "operator_branch" + ? "operator_branch" + : requestedExecutionWorkspaceMode === "agent_default" + ? "adapter_managed" + : "shared_workspace", + strategyType: + executionWorkspace.strategy === "git_worktree" + ? "git_worktree" + : "project_primary", + name: + branchNameForInitialPersistence ?? + issueRef?.identifier ?? + `workspace-${agent.id.slice(0, 8)}`, + status: "active", + cwd: executionWorkspace.cwd, + repoUrl: executionWorkspace.repoUrl, + baseRef: executionWorkspace.repoRef, + branchName: branchNameForInitialPersistence, + providerType: + executionWorkspace.strategy === "git_worktree" + ? "git_worktree" + : "local_fs", + providerRef: executionWorkspace.worktreePath, + lastUsedAt: new Date(), + openedAt: new Date(), + metadata: nextExecutionWorkspaceMetadata, + }) + : null; + } catch (error) { + if (executionWorkspace.created) { + try { + await cleanupExecutionWorkspaceArtifacts({ + workspace: { + id: + reusableExistingExecutionWorkspace?.id ?? + workspaceReuseRequest.requestedExecutionWorkspaceId ?? + `transient-${run.id}`, + cwd: executionWorkspace.cwd, + providerType: + executionWorkspace.strategy === "git_worktree" + ? "git_worktree" + : "local_fs", + providerRef: executionWorkspace.worktreePath, + branchName: executionWorkspace.branchName, + repoUrl: executionWorkspace.repoUrl, + baseRef: executionWorkspace.repoRef, + projectId: resolvedProjectId, + projectWorkspaceId: resolvedProjectWorkspaceId, + sourceIssueId: issueRef?.id ?? null, + metadata: nextExecutionWorkspaceMetadata, + }, + projectWorkspace: { + cwd: resolvedWorkspace.cwd, + cleanupCommand: null, + }, + cleanupCommand: configSnapshot?.cleanupCommand ?? null, + teardownCommand: + configSnapshot?.teardownCommand ?? + projectExecutionWorkspacePolicy?.workspaceStrategy + ?.teardownCommand ?? + null, + recorder: workspaceOperationRecorder, + }); + } catch (cleanupError) { + logger.warn( + { + runId: run.id, + issueId, + executionWorkspaceCwd: executionWorkspace.cwd, + cleanupError: + cleanupError instanceof Error + ? cleanupError.message + : String(cleanupError), + }, + "Failed to cleanup realized execution workspace after persistence failure", + ); + } + } + throw error; + } + await workspaceOperationRecorder.attachExecutionWorkspaceId( + persistedExecutionWorkspace?.id ?? null, + ); + await recordWorkspaceConfigFreshnessOperation({ + recorder: workspaceOperationRecorder, + runId: run.id, + decision: workspaceConfigFreshness, + hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace), + reuseRequested: requestedShouldReuseExisting, + workspaceReused: Boolean(reusedExecutionWorkspace), + configSnapshotRefreshed: + resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, + previousWorkspaceId: + workspaceReuseRequest.requestedExecutionWorkspaceId, + activeWorkspaceId: persistedExecutionWorkspace?.id ?? null, + }); + if ( + reusableExistingExecutionWorkspace && + persistedExecutionWorkspace && + reusableExistingExecutionWorkspace.id !== + persistedExecutionWorkspace.id && + reusableExistingExecutionWorkspace.status === "active" + ) { + await executionWorkspacesSvc.update( + reusableExistingExecutionWorkspace.id, + { + status: "idle", + cleanupReason: null, + }, + ); + } + await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); + const projectRepositoryPaths: string[] = []; + if (executionWorkspace.projectId && resolvedWorkspace.source === "project_primary" && !resolvedWorkspace.baseCwdFallback) { + const repositoryRows = await db.select().from(projectWorkspaces).where(and( + eq(projectWorkspaces.companyId, agent.companyId), + eq(projectWorkspaces.projectId, executionWorkspace.projectId), + )).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id)); + const repositories = await prepareProjectRepositoryWorkspaces({ + cwd: executionWorkspace.cwd, + anchorRepoUrl: executionWorkspace.repoUrl, + workspaces: repositoryRows, + resolveGitAuth: workspaceGitAuthProvider, + }); + const paths = new Map(repositories.map((repo) => [repo.workspaceId, repo.cwd])); + projectRepositoryPaths.push(...repositories.map((repo) => path.relative(executionWorkspace.cwd, repo.cwd))); + if (resolvedWorkspace.workspaceId) paths.set(resolvedWorkspace.workspaceId, executionWorkspace.cwd); + resolvedWorkspace.workspaceHints = resolvedWorkspace.workspaceHints.map((hint) => ({ + ...hint, cwd: paths.get(hint.workspaceId) ?? hint.cwd, + })); + } + if (persistedExecutionWorkspace) { + context.executionWorkspaceId = persistedExecutionWorkspace.id; + await db + .update(heartbeatRuns) + .set({ + contextSnapshot: context, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + } + const environmentAcquireStartedAtMs = Date.now(); + let acquiredEnvironment: Awaited< + ReturnType + >; + try { + await controllerLease.assertOwned(); + const remoteRecovery = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner" + ? runOptions.nativeRestartRecovery : null; + const recoveryWorkspace = remoteRecovery + ? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null; + if (remoteRecovery && (!recoveryWorkspace || remoteRecovery.runId !== run.id || + recoveryWorkspace.providerLeaseId !== remoteRecovery.remote.providerLeaseId || + recoveryWorkspace.remoteCwd !== remoteRecovery.remote.remoteCwd)) { + throw new Error("native_remote_recovery_lease_mismatch"); + } + acquiredEnvironment = await envOrchestrator.acquireForRun({ + companyId: agent.companyId, + selectedEnvironmentId, + localEnvironmentId: localEnvironment.id, + adapterType: agent.adapterType, + adapterConfig: parseObject(agent.adapterConfig), + admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode, + issueId: issueId ?? null, + heartbeatRunId: run.id, + agentId: agent.id, + persistedExecutionWorkspace, + executionWorkspaceSettings: environmentExecutionWorkspaceSettings, + ...(remoteRecovery && recoveryWorkspace ? { reattachRemoteLease: { + leaseId: recoveryWorkspace.leaseId, + providerLeaseId: remoteRecovery.remote.providerLeaseId, + remoteCwd: remoteRecovery.remote.remoteCwd, + } } : {}), + }); + await controllerLease.assertOwned(); + nativeRunnerPreparationSpans.push({ + name: "environment.acquire", + parentName: "task.run", + startedAtMs: environmentAcquireStartedAtMs, + endedAtMs: Date.now(), + attributes: { adapter: agent.adapterType }, + }); + } catch (error) { + nativeRunnerPreparationSpans.push({ + name: "environment.acquire", + parentName: "task.run", + startedAtMs: environmentAcquireStartedAtMs, + endedAtMs: Date.now(), + outcome: "failed", + attributes: { adapter: agent.adapterType }, + }); + throw error; + } + const selectedEnvironment = acquiredEnvironment.environment; + // Defense-in-depth: re-check the actually-acquired environment against the + // execution allowlist. Even if selection were bypassed, a denied (local/ssh/ + // non-k8s) environment FAILS the run here rather than executing untrusted. + const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, { + driver: selectedEnvironment.driver, + provider: + typeof selectedEnvironment.config?.provider === "string" + ? selectedEnvironment.config.provider + : null, + }); + if (!allowlistDecision.allowed) { + logger.error( + { + runId: run.id, + issueId, + agentId: agent.id, + environmentId: selectedEnvironment.id, + deniedDriver: allowlistDecision.deniedDriver, + deniedProvider: allowlistDecision.deniedProvider, + }, + "Execution allowlist denied the resolved environment; failing run", + ); + throw new Error(allowlistDecision.reason); + } + let activeEnvironmentLease = { + environment: acquiredEnvironment.environment, + lease: acquiredEnvironment.lease, + leaseContext: acquiredEnvironment.leaseContext, + }; + const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder( + { + tracer: getStartupTracer(), + incrementCounter: (metric) => { + void incrementToolRuntimeMetricCounter(db, { + companyId: run.companyId, + metric, + }).catch(() => {}); + }, + emitTransportEvent: (event) => { + void (async () => { + await appendRunEvent(run, { + eventType: event.name, + stream: "system", + level: event.dimensions.outcome === "error" ? "warn" : "info", + payload: { ...event.dimensions }, + }); + })().catch(() => {}); + }, + }, + ); + const environmentRealizeStartedAtMs = Date.now(); + let realizationResult: Awaited< + ReturnType + >; + try { + realizationResult = await envOrchestrator.realizeForRun({ + environment: selectedEnvironment, + lease: activeEnvironmentLease.lease, + adapterType: agent.adapterType, + companyId: agent.companyId, + issueId: issueId ?? null, + heartbeatRunId: run.id, + executionWorkspace, + effectiveExecutionWorkspaceMode, + persistedExecutionWorkspace, + duplexObservabilityRecorder, + }); + nativeRunnerPreparationSpans.push({ + name: "environment.workspace.realize", + parentName: "task.run", + startedAtMs: environmentRealizeStartedAtMs, + endedAtMs: Date.now(), + attributes: { driver: selectedEnvironment.driver }, + }); + } catch (error) { + nativeRunnerPreparationSpans.push({ + name: "environment.workspace.realize", + parentName: "task.run", + startedAtMs: environmentRealizeStartedAtMs, + endedAtMs: Date.now(), + outcome: "failed", + attributes: { driver: selectedEnvironment.driver }, + }); + throw error; + } + const environmentRealizeEndedAtMs = Date.now(); + activeEnvironmentLease = { + ...activeEnvironmentLease, + lease: realizationResult.lease, + }; + persistedExecutionWorkspace = + realizationResult.persistedExecutionWorkspace; + // A sandbox realization may materialize or replace the durable workspace + // after the host-side provisioning boundary above. Bind that final ID to + // the issue before dispatch so warm turns reuse the exact same workspace + // and lease scope instead of silently creating a per-run replacement. + await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); + const workspaceRealization = realizationResult.workspaceRealization; + const executionTarget = realizationResult.executionTarget; + // Preserve the host-owned source before adapter context can share lease + // metadata. A later copy-back failure must not adopt a rebound source. + const workspaceRestoreSource = structuredClone(realizationResult.lease); + return { + resolvedWorkspace, + hostExecutionWorkspaceConfig, + latestWorkspaceConfigMetadata, + workspaceConfigFreshness, + workspaceOperationRecorder, + executionWorkspace, + reusedExecutionWorkspace, + resolvedWorkspaceReusePolicy, + persistedExecutionWorkspace, + projectRepositoryPaths, + environmentAcquireStartedAtMs, + selectedEnvironment, + activeEnvironmentLease, + realizationResult, + environmentRealizeEndedAtMs, + workspaceRealization, + executionTarget, + workspaceRestoreSource, + }; +} diff --git a/tests/e2e/composer-stop.spec.ts b/tests/e2e/composer-stop.spec.ts index e5030d6f7bf..46cc882fe14 100644 --- a/tests/e2e/composer-stop.spec.ts +++ b/tests/e2e/composer-stop.spec.ts @@ -205,6 +205,9 @@ for (const adapter of ["process", "paperclip_runner"] as const) { "-e", "console.log('stop fixture ready'); setInterval(() => console.log('working'), 200);", ], + // The child only runs this script; server-only fixture + // loaders must not follow NODE_OPTIONS into the adapter. + env: { NODE_OPTIONS: "" }, graceSec: 1, } : { provider: "codex", model: "gpt-5.1-codex-mini" }, diff --git a/tests/e2e/workspace-bootstrap/README.md b/tests/e2e/workspace-bootstrap/README.md index 29f9458a8b2..3bf25b641e7 100644 --- a/tests/e2e/workspace-bootstrap/README.md +++ b/tests/e2e/workspace-bootstrap/README.md @@ -9,14 +9,13 @@ timeout only when the test explicitly arms a marker inside its repository. From a development worktree, start the foreground instance: ```sh -BOOTSTRAP_FIXTURE_KEY=not-a-provider-key \ -PATH="$PWD/tests/e2e/workspace-bootstrap/bin:$PATH" \ -NODE_ENV=test \ -node cli/node_modules/tsx/dist/cli.mjs cli/src/index.ts test-drive \ - --harness codex --api-key-env BOOTSTRAP_FIXTURE_KEY \ - --company-name 'Workspace Recovery QA' --no-browser +node cli/node_modules/tsx/dist/cli.mjs tests/e2e/workspace-bootstrap/launch.ts ``` +The launcher creates an isolated test-drive, installs the Git shim, and sets a +10-second snapshot deadline after test-drive clears inherited configuration. +Production's longer snapshot deadline would outlast the injected failure. + Set `BOOTSTRAP_REAL_GIT` to the absolute Git executable if it is not `/usr/bin/git`. Do not point this fixture at a normal development or production instance. The seeder checks the loopback host, trusted-local mode, and company diff --git a/tests/e2e/workspace-bootstrap/launch.ts b/tests/e2e/workspace-bootstrap/launch.ts new file mode 100644 index 00000000000..c05925c76ac --- /dev/null +++ b/tests/e2e/workspace-bootstrap/launch.ts @@ -0,0 +1,18 @@ +import path from "node:path"; +import { prepareTestDriveEnvironment, testDriveCommand } from "../../../cli/src/commands/test-drive.js"; + +const options = { + harness: "codex", + apiKeyEnv: "BOOTSTRAP_FIXTURE_KEY", + companyName: "Workspace Recovery QA", + browser: false, +} as const; + +process.env.BOOTSTRAP_FIXTURE_KEY = "not-a-provider-key"; +await prepareTestDriveEnvironment(options); +process.env.NODE_ENV = "test"; +// Test-drive clears PAPERCLIP_* settings. Apply the fixture's short deadline +// after isolation; production snapshots may legitimately take much longer. +process.env.PAPERCLIP_WORKSPACE_GIT_SNAPSHOT_TIMEOUT_MS = "10000"; +process.env.PATH = `${path.join(import.meta.dirname, "bin")}${path.delimiter}${process.env.PATH ?? ""}`; +await testDriveCommand(options); diff --git a/tests/e2e/workspace-bootstrap/recovery.spec.ts b/tests/e2e/workspace-bootstrap/recovery.spec.ts index f8ac233e81f..884a20a8bdc 100644 --- a/tests/e2e/workspace-bootstrap/recovery.spec.ts +++ b/tests/e2e/workspace-bootstrap/recovery.spec.ts @@ -22,11 +22,11 @@ for (const persistent of [false, true]) { if (await announcement.isVisible()) await announcement.click(); await page.getByRole("link", { name: "Tasks", exact: true }).click(); await page.getByRole("button", { name: "New Task", exact: true }).last().click(); - await page.getByPlaceholder("Task title (optional)", { exact: true }).fill(title); - await page.getByRole("button", { name: "Assignee", exact: true }).click(); - await page.getByRole("button", { name: fixture.agentName, exact: true }).click(); + await page.getByRole("textbox", { name: "editable markdown" }).fill(title); + await page.getByRole("button", { name: "Select assignee", exact: true }).click(); + await page.getByRole("option").filter({ hasText: fixture.agentName }).click(); // Let the closing popover unmount before clicking another popover trigger. - await expect(page.getByRole("textbox", { name: "Search assignees...", includeHidden: true })).toHaveCount(0); + await expect(page.getByRole("searchbox", { name: "Search assignees", includeHidden: true })).toHaveCount(0); // The preceding popover's focus restoration can consume the first click. await expect(async () => { if (!await page.getByRole("textbox", { name: "Search projects..." }).isVisible()) { @@ -36,15 +36,21 @@ for (const persistent of [false, true]) { }).toPass({ timeout: 10_000, intervals: [1_000] }); await page.getByRole("button", { name: fixture.projectName, exact: true }).click(); await expect(page.getByRole("textbox", { name: "Search projects...", includeHidden: true })).toHaveCount(0); - await page.getByRole("button", { name: "Create Task", exact: true }).click(); - const taskLink = page.getByRole("complementary").getByRole("link", { name: title, exact: true }); + const created = page.waitForResponse(response => + response.request().method() === "POST" + && new URL(response.url()).pathname === `/api/companies/${fixture.companyId}/issues`, + ); + await page.getByRole("button", { name: "Create task", exact: true }).click(); + const response = await created; + expect(response.ok(), await response.text()).toBeTruthy(); + const task = await response.json(); + expect(task.description).toBe(title); + const taskLink = page.getByRole("complementary").getByRole("link", { name: task.title, exact: true }); await expect(taskLink).toBeVisible(); // Follow the UI's actual persisted link, including across task-tab state. const href = await taskLink.getAttribute("href"); expect(href).toBeTruthy(); await page.goto(new URL(href!, base).href); - const tasks = await api(`/companies/${fixture.companyId}/issues`); - const task = tasks.find((row: { title: string }) => row.title === title); const runs = async () => (await api(`/companies/${fixture.companyId}/heartbeat-runs`)).filter((row: { agentId: string }) => row.agentId === fixture.agentId); await expect.poll(async () => (await runs()).some((row: { errorCode: string }) => row.errorCode === "workspace_git_scan_timeout"), { timeout: 30_000 }).toBe(true); await expect(page.getByText(/Agent resumes in/)).toBeVisible();