diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index c848a781377..055550e6773 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -122,6 +122,8 @@ configuration variables; task bindings cannot override them. When effective run config changes, Paperclip may intentionally skip a saved adapter session, refresh persisted workspace runtime config, replace a reused execution workspace, or avoid reusing a sandbox/environment lease. Fresh execution can lose adapter-specific session, workspace, or sandbox state; correctness of the next run's config takes priority over continuity. Plain environment values affect freshness through value hashes; run result JSON and workspace operation logs expose only the non-sensitive freshness decision categories, without storing secret values, full env maps, provider credentials, or private path details. +A workspace-provider plugin owns the files of the workspaces it prepares, so the run environment only decides where the runner executes. For these workspaces, a changed environment configuration is a refresh, not a replacement: Paperclip records the new workspace fingerprint once and reports the drift in that run only. Other replacement-class changes, such as the repository or base ref, keep the replacement decision. + The saved-session fingerprint covers only inputs that can change what the provider thread saw or may do, or what the native session digest compares before a follow-up turn attaches to its warm Runner. It leaves out the agent config revision row itself (its id and time), agent identity and display fields that tools read live, turn timeouts, the legacy kill grace, permission keys of a provider that is not selected, wake scheduling (`runtimeConfig.heartbeat`), the environment row's update time and selection source, and the environment's release, log-streaming, and RPC-timeout settings. Lifecycle inputs stay in: the lifecycle mode, the agent and environment warm idle windows, and the environment's lease reuse, because they become `session.lifecyclePolicy`, and a kept session with a different policy cannot attach to its warm Runner. `server/src/services/heartbeat/session-fingerprint-inputs.ts` classifies every input with the reason. Its tests fail when an agent revision field, a runner config schema field, a provider permission key, or an environment config key appears without a classification, when a new field reaches a hashed category, or when an excluded runner key reaches the native execution input. An unknown adapter or runtime config key is hashed. A session saved with the previous fingerprint composition is accepted once for the same inputs, so an upgrade does not discard saved sessions. An explicit `reuse_existing` Git workspace keeps its recorded source repository, including after its project-workspace row is deleted or the project primary changes. The server verifies the original company, project, repository origin, and worktree registration before reuse. It does not attach a deleted association to the new primary. A conflicting explicit project-workspace selection, missing source identity, or an unverified legacy local checkout blocks the run. Restore the original source. Before intentionally clearing the existing-workspace binding to choose a new repository, review and preserve its retained work. Missing worktrees may be reconstructed only from the verified original source; current project defaults are not a fallback. diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index 42aa9cf805f..bb6242fd2df 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -1713,6 +1713,62 @@ describe("effective run execution workspace config freshness", () => { expect(decision.changedCategories).toContain(category); }); + describe("provider-owned storage", () => { + const pluginEnvironment = (template: string) => ({ + selectedEnvironmentId: "environment-1", + driver: "plugin", + config: { pluginKey: "example.sandbox", driverKey: "tasks", driverConfig: { template } }, + }); + + it("records environment drift as a refresh that persists the new fingerprint", () => { + const base = buildWorkspaceConfigMetadata({ environment: pluginEnvironment("template-a") }); + const next = buildWorkspaceConfigMetadata({ environment: pluginEnvironment("template-b") }); + + const decision = resolveExecutionWorkspaceConfigFreshness({ + hasExistingWorkspace: true, + existingWorkspaceMetadata: persistedWorkspaceConfigFingerprint(base), + nextMetadata: next, + providerOwnedStorage: true, + }); + const policy = resolveExecutionWorkspaceReuseProvisioningPolicy({ + requestedShouldReuseExisting: true, + workspaceConfigFreshness: decision, + }); + + expect(decision).toMatchObject({ action: "refresh", shouldReuseExisting: true, changedCategories: ["environment"] }); + expect(policy).toMatchObject({ shouldRestoreExistingWorkspace: true, shouldPersistLatestWorkspaceConfigMetadata: true }); + }); + + it("still replaces on environment drift when Core owns the workspace", () => { + const decision = resolveExecutionWorkspaceConfigFreshness({ + hasExistingWorkspace: true, + existingWorkspaceMetadata: persistedWorkspaceConfigFingerprint( + buildWorkspaceConfigMetadata({ environment: pluginEnvironment("template-a") }), + ), + nextMetadata: buildWorkspaceConfigMetadata({ environment: pluginEnvironment("template-b") }), + }); + + expect(decision.action).toBe("replace"); + }); + + it("still replaces when another replacement-class category drifts alongside the environment", () => { + const decision = resolveExecutionWorkspaceConfigFreshness({ + hasExistingWorkspace: true, + existingWorkspaceMetadata: persistedWorkspaceConfigFingerprint( + buildWorkspaceConfigMetadata({ environment: pluginEnvironment("template-a") }), + ), + nextMetadata: buildWorkspaceConfigMetadata({ + environment: pluginEnvironment("template-b"), + repoRef: "origin/release", + }), + providerOwnedStorage: true, + }); + + expect(decision.action).toBe("replace"); + expect(decision.changedCategories).toEqual(expect.arrayContaining(["environment", "repo"])); + }); + }); + it("keeps replacement-class drift visible when explicit reuse restores the old workspace", () => { const base = buildWorkspaceConfigMetadata(); const next = buildWorkspaceConfigMetadata({ diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index d2039d644f3..f86ac513682 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -969,10 +969,11 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true }, { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true, refuse: true }, { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true, restart: true }, - ])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare, verify: $verify, refuse: $refuse, restart: $restart)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare, verify, refuse, restart }) => { + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: false, environmentDrift: true }, + ])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare, verify: $verify, refuse: $refuse, restart: $restart, environmentDrift: $environmentDrift)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare, verify, refuse, restart, environmentDrift }) => { const { input, source, issue } = await fixture(true); if (portable) input.agent.adapterType = "paperclip_runner"; - const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : refuse ? "-refuse" : restart ? "-restart" : verify ? "-verify" : ""}`; + const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : refuse ? "-refuse" : restart ? "-restart" : verify ? "-verify" : environmentDrift ? "-environment-drift" : ""}`; const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: "test-storage", version: "1.0.0", apiVersion: 1, status: "ready", categories: ["workspace"], manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Storage", description: "Storage", author: "Test", categories: ["workspace"], workspaceProvider: { freshWorkspaceVersion: 1, finalizationVersion: 1, workspaceVerificationVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"], @@ -1195,6 +1196,27 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace const [task] = await db.select().from(issues).where(eq(issues.id, issue.id)); const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, task.executionWorkspaceId!)); expect(workspace).toMatchObject({ providerType: "plugin", cwd: null, mode: defaultMode, strategyType: "git_worktree", branchName: fresh ? null : fromDefault ? "WSP-1-prepare-this-workspace" : "test-WSP-1" }); + if (environmentDrift) { + const fingerprintOf = (row: typeof workspace) => (row.metadata?.configFingerprint as { workspaceHash: string }).workspaceHash; + const freshnessFor = async (runId: string) => (await db.select().from(workspaceOperations) + .where(eq(workspaceOperations.heartbeatRunId, runId))).filter(operation => operation.phase === "workspace_config_freshness"); + await db.update(environments).set({ config: { pluginKey, driverKey: "tasks", driverConfig: { template: "next" } } }) + .where(eq(environments.id, environment.id)); + input.environment.selectedEnvironmentForConfig = await environmentService(db).getById(environment.id); + await reuse(input, toExecutionWorkspace(workspace)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached execution handoff"); + const drifted = await freshnessFor(input.run.id); + expect(drifted).toHaveLength(1); + expect(drifted[0].metadata).toMatchObject({ action: "refresh", changedCategories: ["environment"], workspaceReused: true }); + const [refreshed] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace.id)); + expect(fingerprintOf(refreshed)).not.toBe(fingerprintOf(workspace)); + await reuse(input, toExecutionWorkspace(refreshed)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached execution handoff"); + expect(await freshnessFor(input.run.id)).toEqual([]); + const [settled] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace.id)); + expect(fingerprintOf(settled)).toBe(fingerprintOf(refreshed)); + return; + } if (portable) { const finalization = { runId: input.run.id, agentId: input.agent.id }; const plan = await resolvePluginWorkspaceFinalization(db, workspace, finalization); diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index c773418f22b..e3eb9b09da7 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -35,6 +35,7 @@ import { reconcileReusedExecutionWorkspaceProjectWorkspaceId, recordWorkspaceConfigFreshnessOperation, prepareProjectRepositoryWorkspaces, + WORKSPACE_CONFIG_FINGERPRINT_METADATA_KEY, } from "./workspaces.js"; import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; import fs from "node:fs/promises"; @@ -863,6 +864,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace reusableExistingExecutionWorkspace?.metadata ?? null, inferredMetadata: inferredExistingWorkspaceConfigMetadata, nextMetadata: latestWorkspaceConfigMetadata, + providerOwnedStorage: Boolean(pluginPrepared), }, ); const workspaceReuseProvisioningPolicy = @@ -1182,9 +1184,13 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace ); } } + const coreConfigFingerprint = baseExecutionWorkspaceMetadata?.[WORKSPACE_CONFIG_FINGERPRINT_METADATA_KEY]; const nextExecutionWorkspaceMetadata = { ...baseExecutionWorkspaceMetadata, ...(pluginPrepared?.metadata ?? {}), + ...(pluginPrepared && coreConfigFingerprint !== undefined + ? { [WORKSPACE_CONFIG_FINGERPRINT_METADATA_KEY]: coreConfigFingerprint } + : {}), ...(taskWorkspaceOwner && !pluginPrepared ? { runnerTaskWorkspaceOwner: taskWorkspaceOwner } : {}), ...(providerFresh && pluginTaskWorkspace && !pluginPrepared ? { runnerTaskFreshWorkspace: providerFresh } : {}), ...(copyOnlyProvisioning ? { [CONTROLLER_PROVISION_COPY_ONLY]: true } : {}), diff --git a/server/src/services/heartbeat/workspaces.ts b/server/src/services/heartbeat/workspaces.ts index 8f28452360e..3a65e5ebf9b 100644 --- a/server/src/services/heartbeat/workspaces.ts +++ b/server/src/services/heartbeat/workspaces.ts @@ -2128,7 +2128,7 @@ const PAPERCLIP_SESSION_METADATA_KEYS = new Set([ SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY, ]); -const WORKSPACE_CONFIG_FINGERPRINT_METADATA_KEY = "configFingerprint"; +export const WORKSPACE_CONFIG_FINGERPRINT_METADATA_KEY = "configFingerprint"; const EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES = [ "adapter", @@ -3034,6 +3034,7 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { existingWorkspaceMetadata: Record | null | undefined; inferredMetadata?: EffectiveRunWorkspaceConfigMetadata | null; nextMetadata: EffectiveRunWorkspaceConfigMetadata | null; + providerOwnedStorage?: boolean; }): ExecutionWorkspaceConfigFreshnessDecision { if (!input.hasExistingWorkspace) { return { @@ -3138,8 +3139,10 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { previous: previous.categoryFingerprints, next: input.nextMetadata.categoryFingerprints, }); - const replacementRequired = changedCategories.some((category) => - WORKSPACE_REPLACEMENT_CONFIG_CATEGORIES.has(category), + const replacementRequired = changedCategories.some( + (category) => + WORKSPACE_REPLACEMENT_CONFIG_CATEGORIES.has(category) && + !(input.providerOwnedStorage && category === "environment"), ); const action: WorkspaceConfigFreshnessDecisionAction = replacementRequired ? "replace"