diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 0c9e4eb43f5..38a217d3579 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1905,6 +1905,18 @@ corrupt, or unbound sources block the import. Publication still requires an independent download and restore of the new encrypted snapshot. The old folder remains absent, and the importer does not change the original restore point. +Shared physical folders require Cloud's offline enrollment maintenance process. +A task-level hold cannot fence another workspace record pointing at the same +folder. Maintenance verifies that legacy deployments have stopped and that no +runs are active or queued, then imports nested task directories before parent +roots. A parent can omit a runtime-created nested task folder only after an +independent restore and source comparison prove that its committed snapshot +covers those bytes. The parent snapshot records that dependency, and retention +pins the child snapshot until the parent is deleted. Changed or unavailable +nested data blocks publication. Ordinary user folders remain in the archive. +Interrupted maintenance resumes the same import journals; completed records +leave the pending set. Original folders remain on the retained volume. + Supported sources include empty task folders, repository URLs, and managed mutable repositories. Authorized mentioned projects attach all their configured Git repositories as writable copies under `.paperclip-repositories/` in the task diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 65c7dd3d351..9bf24ecb59c 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -23,8 +23,10 @@ encrypted task-directory restoration, Git restoration recipes, and legacy branch-deletion receipts. PostgreSQL and real-file tests prove cold restoration of saved bytes, index state, and unpushed commits, including after pruning the old Git object store. Invalid or unavailable retained data still blocks import. -Shared-root migration and recovery-obligation reconciliation remain separate -gaps; this slice does not make a whole tenant eligible for enrollment. +The next slice adds leaf-first offline migration of overlapping folders, verified +nested snapshot dependencies, interruption recovery, and dependency retention. +Recovery-obligation reconciliation remains a separate gap; these slices do not +yet make the whole tenant eligible for enrollment. - Import existing files or a verified retained source. A missing folder alone is not permission to publish an empty workspace or choose a newer Git branch. diff --git a/packages/db/src/migrations/0427_ordinary_xavin.sql b/packages/db/src/migrations/0427_ordinary_xavin.sql new file mode 100644 index 00000000000..f99ec2f2af5 --- /dev/null +++ b/packages/db/src/migrations/0427_ordinary_xavin.sql @@ -0,0 +1 @@ +CREATE INDEX "execution_workspace_snapshots_imported_children_idx" ON "execution_workspace_snapshots" USING gin (("metadata"->'importedWorkspaceChildren') jsonb_path_ops) WHERE "execution_workspace_snapshots"."metadata" ? 'importedWorkspaceChildren'; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0421_snapshot.json b/packages/db/src/migrations/meta/0427_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0421_snapshot.json rename to packages/db/src/migrations/meta/0427_snapshot.json index b48d455eb8f..5ea62c26145 100644 --- a/packages/db/src/migrations/meta/0421_snapshot.json +++ b/packages/db/src/migrations/meta/0427_snapshot.json @@ -1,6 +1,6 @@ { - "id": "deb3f1b9-6eb7-447f-adf2-dee69f943c59", - "prevId": "8f75a1bb-32d9-4aed-9f93-134a6ae71f79", + "id": "dd860bff-55de-4849-8787-58b9de729258", + "prevId": "a1fcb4c6-40e7-47a6-b3f2-c43cd9bf332b", "version": "7", "dialect": "postgresql", "tables": { @@ -2233,6 +2233,74 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.agent_file_roots": { + "name": "agent_file_roots", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_file_roots_company_id_agent_id_agents_company_id_id_fk": { + "name": "agent_file_roots_company_id_agent_id_agents_company_id_id_fk", + "tableFrom": "agent_file_roots", + "tableTo": "agents", + "columnsFrom": [ + "company_id", + "agent_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_file_roots_company_id_agent_id_pk": { + "name": "agent_file_roots_company_id_agent_id_pk", + "columns": [ + "company_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.agent_identity_keys": { "name": "agent_identity_keys", "schema": "", @@ -28757,6 +28825,327 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.execution_admission_project_charges": { + "name": "execution_admission_project_charges", + "schema": "", + "columns": { + "admission_id": { + "name": "admission_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "execution_admission_project_charges_project_idx": { + "name": "execution_admission_project_charges_project_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_admission_project_charges_company_admission_fk": { + "name": "execution_admission_project_charges_company_admission_fk", + "tableFrom": "execution_admission_project_charges", + "tableTo": "execution_admissions", + "columnsFrom": [ + "company_id", + "admission_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_admission_project_charges_company_project_fk": { + "name": "execution_admission_project_charges_company_project_fk", + "tableFrom": "execution_admission_project_charges", + "tableTo": "projects", + "columnsFrom": [ + "company_id", + "project_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_admission_project_charges_admission_id_project_id_pk": { + "name": "execution_admission_project_charges_admission_id_project_id_pk", + "columns": [ + "admission_id", + "project_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_admissions": { + "name": "execution_admissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "primary_project_id": { + "name": "primary_project_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "context_revision": { + "name": "context_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settlement_id": { + "name": "settlement_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "settled_at": { + "name": "settled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_admissions_company_id_uq": { + "name": "execution_admissions_company_id_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_admissions_run_attempt_uq": { + "name": "execution_admissions_run_attempt_uq", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_admissions_live_run_uq": { + "name": "execution_admissions_live_run_uq", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"execution_admissions\".\"settled_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_admissions_company_id_companies_id_fk": { + "name": "execution_admissions_company_id_companies_id_fk", + "tableFrom": "execution_admissions", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "execution_admissions_run_id_heartbeat_runs_id_fk": { + "name": "execution_admissions_run_id_heartbeat_runs_id_fk", + "tableFrom": "execution_admissions", + "tableTo": "heartbeat_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "execution_admissions_attempt_check": { + "name": "execution_admissions_attempt_check", + "value": "\"execution_admissions\".\"attempt\" > 0" + }, + "execution_admissions_settlement_check": { + "name": "execution_admissions_settlement_check", + "value": "(\"execution_admissions\".\"settlement_id\" is null) = (\"execution_admissions\".\"settled_at\" is null)" + } + }, + "isRLSEnabled": false + }, + "public.project_task_execution_policies": { + "name": "project_task_execution_policies", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "max_concurrent_tasks": { + "name": "max_concurrent_tasks", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "project_task_execution_policies_company_project_fk": { + "name": "project_task_execution_policies_company_project_fk", + "tableFrom": "project_task_execution_policies", + "tableTo": "projects", + "columnsFrom": [ + "company_id", + "project_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "project_task_execution_policies_maximum_check": { + "name": "project_task_execution_policies_maximum_check", + "value": "\"project_task_execution_policies\".\"max_concurrent_tasks\" is null or \"project_task_execution_policies\".\"max_concurrent_tasks\" > 0" + } + }, + "isRLSEnabled": false + }, "public.execution_workspace_repositories": { "name": "execution_workspace_repositories", "schema": "", @@ -29931,6 +30320,22 @@ "concurrently": false, "method": "btree", "with": {} + }, + "execution_workspace_snapshots_imported_children_idx": { + "name": "execution_workspace_snapshots_imported_children_idx", + "columns": [ + { + "expression": "(\"metadata\"->'importedWorkspaceChildren') jsonb_path_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_workspace_snapshots\".\"metadata\" ? 'importedWorkspaceChildren'", + "concurrently": false, + "method": "gin", + "with": {} } }, "foreignKeys": {}, @@ -50412,6 +50817,12 @@ "primaryKey": false, "notNull": false }, + "created_by_run_id": { + "name": "created_by_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, "dispatch_fingerprint": { "name": "dispatch_fingerprint", "type": "text", @@ -62374,6 +62785,12 @@ "primaryKey": false, "notNull": false }, + "cleanup_verified_at": { + "name": "cleanup_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index b8a5f20dd9a..75ac82c14f3 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2976,6 +2976,13 @@ "when": 1793733446322, "tag": "0426_absent_guardian", "breakpoints": true + }, + { + "idx": 427, + "version": "7", + "when": 1793736361959, + "tag": "0427_ordinary_xavin", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/execution_workspace_snapshots.ts b/packages/db/src/schema/execution_workspace_snapshots.ts index f5f155c5d6f..65e33229a58 100644 --- a/packages/db/src/schema/execution_workspace_snapshots.ts +++ b/packages/db/src/schema/execution_workspace_snapshots.ts @@ -1,4 +1,5 @@ import { bigint, index, integer, jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core"; +import { sql } from "drizzle-orm"; /** Durable object ledger deliberately survives deletion of its source records. */ export const executionWorkspaceSnapshots = pgTable("execution_workspace_snapshots", { @@ -27,4 +28,7 @@ export const executionWorkspaceSnapshots = pgTable("execution_workspace_snapshot generationUniqueIdx: uniqueIndex("execution_workspace_snapshots_generation_uniq").on(table.companyId, table.executionWorkspaceId, table.generation), workspaceIdx: index("execution_workspace_snapshots_workspace_idx").on(table.companyId, table.executionWorkspaceId), cleanupIdx: index("execution_workspace_snapshots_cleanup_idx").on(table.status, table.deleteAfter), + importedChildrenIdx: index("execution_workspace_snapshots_imported_children_idx") + .using("gin", sql`(${table.metadata}->'importedWorkspaceChildren') jsonb_path_ops`) + .where(sql`${table.metadata} ? 'importedWorkspaceChildren'`), })); diff --git a/server/src/__tests__/ephemeral-migration.test.ts b/server/src/__tests__/ephemeral-migration.test.ts index 69733b6eb8d..a3ef74bd1d2 100644 --- a/server/src/__tests__/ephemeral-migration.test.ts +++ b/server/src/__tests__/ephemeral-migration.test.ts @@ -10,8 +10,11 @@ import { randomBytes, randomUUID } from "node:crypto"; import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; +import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "../services/remote-workspace-snapshot-storage.js"; +import { restoreRemoteWorkspaceArchive } from "../services/remote-workspace-snapshot-archive.js"; import { migrateEphemeralServerData } from "../maintenance/ephemeral-migration-data.js"; import { agentFileStore } from "../services/agent-file-store.js"; import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js"; @@ -35,6 +38,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll home = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-migration-")); await s3.send(new CreateBucketCommand({ Bucket: bucket })); vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", process.env.PAPERCLIP_TEST_S3_ENDPOINT!); @@ -43,6 +47,10 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll }); afterEach(async () => { await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(workspaceOperations).where(eq(workspaceOperations.companyId, companyId)); + await db.delete(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.companyId, companyId)); + await db.delete(issues).where(eq(issues.companyId, companyId)); + await db.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId)); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); await db.delete(agents).where(eq(agents.companyId, companyId)); await db.delete(companies).where(eq(companies.id, companyId)); @@ -90,6 +98,33 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll await s3.send(new PutObjectCommand({ Bucket: bucket, Key: file.objectKey, Body: "corrupt" })); await expect(verifyAgentFileObjects(db, getStorageService(), AbortSignal.timeout(30_000))).rejects.toThrow(/agent_file_object_(size|digest)_mismatch/); }, 120_000); + it("migrates overlapping task folders through maintenance and restores both from S3 after volume loss", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32)); + const parent = path.join(home, "project"), child = path.join(parent, ".paperclip/worktrees/task"); + await fs.mkdir(child, { recursive: true }); + const fixtures = [{ cwd: parent, file: "project.txt", bytes: "project work" }, { cwd: child, file: "task.txt", bytes: "nested task work" }]; + const ids: string[] = []; + for (const fixture of fixtures) { + await fs.writeFile(path.join(fixture.cwd, fixture.file), fixture.bytes); + const [issue] = await db.insert(issues).values({ companyId, title: fixture.file }).returning(); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issue!.id, + name: fixture.file, mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: fixture.cwd }).returning(); + ids.push(workspace!.id); + await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issue!.id)); + } + expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 2 } }); + await fs.rm(home, { recursive: true }); home = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-successor-")); + vi.stubEnv("PAPERCLIP_HOME", home); + for (const [index, workspaceId] of ids.entries()) { + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); + const authority = readRemoteWorkspaceAuthority(workspace!.metadata)!; + const reference = await getWorkspaceSnapshot(db, { companyId, workspaceId, snapshotId: authority.snapshotId! }); + const archivePath = path.join(home, `${index}.pcws`), targetDir = path.join(home, `restored-${index}`); + await loadWorkspaceSnapshotArchive({ storage: getStorageService(), reference: reference!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, fixtures[index]!.file), "utf8")).toBe(fixtures[index]!.bytes); + } + }, 120_000); const entry = fileURLToPath(new URL("../../dist-runtime/ephemeral-migration.mjs", import.meta.url)); it.skipIf(!existsSync(entry))("runs the actual bundled maintenance entry with plain Node and exits cleanly", async () => { await agent(); diff --git a/server/src/maintenance/ephemeral-migration-data.ts b/server/src/maintenance/ephemeral-migration-data.ts index 17c713b0f12..eda8e547d6e 100644 --- a/server/src/maintenance/ephemeral-migration-data.ts +++ b/server/src/maintenance/ephemeral-migration-data.ts @@ -8,6 +8,7 @@ import { skillInstallationMaintenance } from "../services/skill-installation-mai import { skillPackageMaintenance } from "../services/skill-package-maintenance.js"; import { hasUnsettledLocalInstructionCopies } from "../services/agent-instruction-working-copies.js"; import { providerTraceStore } from "../services/provider-trace-store.js"; +import { migrateWorkspaceTrees } from "../services/remote-workspace-migration-batch.js"; /** Runs only in the maintenance process after Cloud has confirmed all legacy * deployments stopped. Each owner retains its own migration/publication rules; @@ -23,10 +24,15 @@ export async function migrateEphemeralServerData(db: Db, input: { if (await hasUnsettledLocalInstructionCopies(db)) throw new Error("local_instruction_copies_require_settlement"); const before = await readEphemeralServerDataEligibility(db); const unsupported = before.blockers.filter(blocker => ![ - "agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration", + "agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration", "workspaces_require_migration", ].includes(blocker)); if (unsupported.length) throw new Error(unsupported[0]); + phase("workspaces"); + const storage = getStorageService(); + if (storage.provider !== "s3") throw new Error("shared_object_storage_required"); + counts.workspaces = await migrateWorkspaceTrees(db, { storage, signal, phase }); + phase("agent_files"); let after: string | undefined; do { @@ -77,8 +83,6 @@ export async function migrateEphemeralServerData(db: Db, input: { } phase("verify_agent_objects"); - const storage = getStorageService(); - if (storage.provider !== "s3") throw new Error("shared_object_storage_required"); counts.verifiedAgentObjects = await verifyAgentFileObjects(db, storage, signal); phase("verify_skill_versions"); after = undefined; diff --git a/server/src/services/remote-workspace-migration-batch.ts b/server/src/services/remote-workspace-migration-batch.ts new file mode 100644 index 00000000000..baa3f090739 --- /dev/null +++ b/server/src/services/remote-workspace-migration-batch.ts @@ -0,0 +1,34 @@ +import { and, asc, desc, isNotNull, sql } from "drizzle-orm"; +import { executionWorkspaces, type Db } from "@paperclipai/db"; +import type { StorageService } from "../storage/types.js"; +import { offlineWorkspaceMigrationBlockers } from "./remote-workspace-migration-tree.js"; +import { migrateRemoteWorkspace } from "./remote-workspace-migration.js"; + +/** The Cloud maintenance process owns the stop-all-deployments precondition. + * Imported records leave the pending set, so a restart resumes the same owner + * journals without moving the task binding or importing over remote authority. */ +export async function migrateWorkspaceTrees(db: Db, input: { + storage: StorageService; signal: AbortSignal; phase: (name: string) => void; +}) { + const blockers = await offlineWorkspaceMigrationBlockers(db); + if (blockers.length) throw new Error(blockers[0]); + let migrated = 0; + for (;;) { + input.signal.throwIfAborted(); + // Match the final ephemeral-storage contract, not just the importer's + // supported subset. Silently skipping an unsupported provider or unsettled + // lifecycle state would abandon files when the original volume is detached. + const rows = await db.select({ companyId: executionWorkspaces.companyId, workspaceId: executionWorkspaces.id }) + .from(executionWorkspaces).where(and(isNotNull(executionWorkspaces.cwd), + sql`${executionWorkspaces.providerType} <> 'plugin'`, + sql`coalesce(${executionWorkspaces.metadata}->'remoteWorkspace'->>'schema', '') <> 'paperclip.remote-workspace.v1'`, + )).orderBy(desc(sql`length(${executionWorkspaces.cwd})`), asc(executionWorkspaces.id)).limit(25); + if (!rows.length) return migrated; + for (const workspace of rows) { + input.signal.throwIfAborted(); + input.phase(`workspace:${workspace.workspaceId}`); + await migrateRemoteWorkspace({ db, storage: input.storage, ...workspace, apply: true, writersStopped: true, signal: input.signal }); + migrated++; + } + } +} diff --git a/server/src/services/remote-workspace-migration-tree.ts b/server/src/services/remote-workspace-migration-tree.ts new file mode 100644 index 00000000000..3c7deed9113 --- /dev/null +++ b/server/src/services/remote-workspace-migration-tree.ts @@ -0,0 +1,148 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { and, eq, inArray, ne, sql } from "drizzle-orm"; +import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, workspaceOperations, type Db } from "@paperclipai/db"; +import type { StorageService } from "../storage/types.js"; +import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; +import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; + +type ReadDb = Pick; +type Workspace = typeof executionWorkspaces.$inferSelect; +export type ImportedWorkspaceChild = { + workspaceId: string; snapshotId: string; operationId: string; relativePath: string; manifestDigest: string; +}; +const overlaps = (a: string, b: string) => a === b || a.startsWith(`${b}/`) || b.startsWith(`${a}/`); +const rootOf = (workspace: Pick) => workspace.cwd ?? workspace.providerRef; + +// Missing historical paths still resolve through their nearest surviving +// parent, so a symlinked ancestor cannot hide an alias from the writer fence. +async function physicalPath(value: string): Promise { + let current = path.resolve(value); const missing: string[] = []; + for (;;) { + try { return path.join(await fs.realpath(current), ...missing.reverse()); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" || current === path.dirname(current)) throw error; + missing.push(path.basename(current)); current = path.dirname(current); + } + } +} + +export function validateImportedWorkspaceChildren(raw: unknown): ImportedWorkspaceChild[] { + if (!Array.isArray(raw) || raw.some(child => !child || typeof child !== "object" + || typeof child.relativePath !== "string" || !child.relativePath.startsWith(".paperclip/worktrees/") + || child.relativePath.split("/").some((part: string) => !part || part === "." || part === "..") + || !/^[a-f0-9-]{36}$/.test(child.workspaceId) || !/^[a-f0-9-]{36}$/.test(child.snapshotId) + || !/^[a-f0-9-]{36}$/.test(child.operationId) || !/^[a-f0-9]{64}$/.test(child.manifestDigest))) + throw new Error("nested_workspace_coverage_invalid"); + return raw; +} + +export async function offlineWorkspaceMigrationBlockers(db: ReadDb) { + if (process.env.PAPERCLIP_EPHEMERAL_MIGRATION !== "true") return ["offline_maintenance_required"]; + const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(inArray(heartbeatRuns.status, ["running", "queued"])).limit(1); + return active ? ["active_execution_requires_drain"] : []; +} + +/** Only the maintenance entry point can acknowledge that every old deployment + * has stopped. A task-level migration hold cannot fence other path aliases. */ +export async function workspaceMigrationLayoutBlockers(db: ReadDb, workspace: Workspace, writersStopped = false) { + if (writersStopped) return offlineWorkspaceMigrationBlockers(db); + const cwd = rootOf(workspace); + if (!cwd) return []; + const local = await db.select({ id: executionWorkspaces.id, cwd: executionWorkspaces.cwd, + providerRef: executionWorkspaces.providerRef }).from(executionWorkspaces).where(and( + eq(executionWorkspaces.companyId, workspace.companyId), ne(executionWorkspaces.id, workspace.id), + inArray(executionWorkspaces.providerType, ["local_fs", "git_worktree"]), + )); + const root = await physicalPath(cwd); + for (const row of local) if (rootOf(row) && overlaps(root, await physicalPath(rootOf(row)!))) + return ["overlapping_workspace_requires_maintenance"]; + const migrated = await db.select({ cwd: workspaceOperations.cwd }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.phase, "remote_workspace_migration"), + eq(workspaceOperations.status, "succeeded"), ne(workspaceOperations.executionWorkspaceId, workspace.id), + )); + for (const row of migrated) if (row.cwd && (await physicalPath(row.cwd)).startsWith(`${root}/.paperclip/worktrees/`)) + return ["overlapping_workspace_requires_maintenance"]; + return []; +} + +/** Runtime-created nested task folders can be omitted only after an exact + * committed import covers their current bytes. Ordinary user folders stay in + * the parent archive. The snapshot owner retains the resulting dependencies. */ +export async function prepareNestedWorkspaceImports(input: { + db: Db; storage: StorageService; workspace: Workspace; scratch: string; signal?: AbortSignal; + recordedChildren?: ImportedWorkspaceChild[]; +}): Promise { + const sourceRoot = rootOf(input.workspace); + if (!sourceRoot) return []; + if (input.recordedChildren && !input.recordedChildren.length) return []; + const root = await physicalPath(sourceRoot); + const rows = await input.db.select({ operation: workspaceOperations, snapshot: executionWorkspaceSnapshots }) + .from(workspaceOperations).innerJoin(executionWorkspaceSnapshots, and( + eq(executionWorkspaceSnapshots.companyId, workspaceOperations.companyId), + eq(executionWorkspaceSnapshots.executionWorkspaceId, workspaceOperations.executionWorkspaceId), + eq(executionWorkspaceSnapshots.sourceOperationId, workspaceOperations.id), + )).where(and(eq(workspaceOperations.companyId, input.workspace.companyId), + ne(workspaceOperations.executionWorkspaceId, input.workspace.id), eq(workspaceOperations.phase, "remote_workspace_migration"), + eq(workspaceOperations.status, "succeeded"), inArray(executionWorkspaceSnapshots.status, ["committed", "superseded"]), + sql`${workspaceOperations.metadata}->>'snapshotId' = ${executionWorkspaceSnapshots.id}::text`, + )); + const located = []; + for (const row of rows) { + if (!row.operation.cwd || input.recordedChildren && !input.recordedChildren.some(child => child.operationId === row.operation.id)) continue; + located.push({ ...row, sourcePath: await physicalPath(row.operation.cwd) }); + } + const candidates = located.filter(row => row.sourcePath.startsWith(`${root}/.paperclip/worktrees/`)) + .sort((a, b) => a.sourcePath.length - b.sourcePath.length || a.operation.id.localeCompare(b.operation.id)); + const covered: ImportedWorkspaceChild[] = []; + for (const { operation, snapshot, sourcePath } of candidates) { + input.signal?.throwIfAborted(); + const relativePath = path.relative(root, sourcePath).split(path.sep).join("/"); + if (covered.some(item => relativePath === item.relativePath || relativePath.startsWith(`${item.relativePath}/`))) continue; + const entry = await fs.lstat(sourcePath).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (!entry) { if (input.recordedChildren) throw new Error("nested_workspace_source_changed"); continue; } + if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("nested_workspace_source_changed"); + await verifyCoveredWorkspace({ ...input, sourcePath, snapshot, operationId: operation.id, + expectedDigest: operation.metadata?.manifestDigest, depth: 0 }); + covered.push({ workspaceId: snapshot.executionWorkspaceId, snapshotId: snapshot.id, operationId: operation.id, + relativePath, manifestDigest: snapshot.manifestDigest! }); + } + if (input.recordedChildren && !isDeepStrictEqual(covered, input.recordedChildren)) throw new Error("nested_workspace_scope_changed"); + return covered; +} + +async function verifyCoveredWorkspace(input: { + db: Db; storage: StorageService; sourcePath: string; snapshot: typeof executionWorkspaceSnapshots.$inferSelect; + operationId: string; expectedDigest: unknown; scratch: string; signal?: AbortSignal; depth: number; +}) { + input.signal?.throwIfAborted(); + if (input.depth > 32) throw new Error("nested_workspace_coverage_invalid"); + const { snapshot } = input; + const reference = await getWorkspaceSnapshot(input.db, { companyId: snapshot.companyId, + workspaceId: snapshot.executionWorkspaceId, snapshotId: snapshot.id }); + if (!reference || reference.manifestDigest !== input.expectedDigest || snapshot.sourceOperationId !== input.operationId) + throw new Error("nested_workspace_snapshot_changed"); + const children = validateImportedWorkspaceChildren(snapshot.metadata?.importedWorkspaceChildren ?? []); + const area = await fs.mkdtemp(path.join(input.scratch, "covered-")); + for (const child of children) { + const [saved] = await input.db.select({ operation: workspaceOperations, snapshot: executionWorkspaceSnapshots }) + .from(executionWorkspaceSnapshots).innerJoin(workspaceOperations, and( + eq(workspaceOperations.companyId, executionWorkspaceSnapshots.companyId), eq(workspaceOperations.id, executionWorkspaceSnapshots.sourceOperationId), + )).where(and(eq(executionWorkspaceSnapshots.companyId, snapshot.companyId), eq(executionWorkspaceSnapshots.id, child.snapshotId), + eq(executionWorkspaceSnapshots.executionWorkspaceId, child.workspaceId), eq(workspaceOperations.id, child.operationId), + eq(workspaceOperations.status, "succeeded"), eq(workspaceOperations.phase, "remote_workspace_migration"), + inArray(executionWorkspaceSnapshots.status, ["committed", "superseded"]))); + const sourcePath = path.join(input.sourcePath, child.relativePath); + if (!saved?.operation.cwd || await physicalPath(saved.operation.cwd) !== sourcePath) throw new Error("nested_workspace_coverage_invalid"); + await verifyCoveredWorkspace({ ...input, sourcePath, snapshot: saved.snapshot, operationId: child.operationId, + expectedDigest: child.manifestDigest, scratch: area, depth: input.depth + 1 }); + } + const archivePath = path.join(area, "saved.pcws"), restored = path.join(area, "restored"); + await loadWorkspaceSnapshotArchive({ storage: input.storage, reference, targetPath: archivePath, signal: input.signal }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir: restored, expectedManifestDigest: reference.manifestDigest, signal: input.signal }); + const current = await captureRemoteWorkspaceDirectory({ cwd: input.sourcePath, archivePath: path.join(area, "current.pcws"), + exclude: children.map(child => child.relativePath), signal: input.signal }); + if (current.manifestDigest !== reference.manifestDigest) throw new Error("nested_workspace_source_changed"); +} diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index e2345e3cb88..bce64e9e66d 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -8,13 +8,14 @@ import { createServer, type Server } from "node:http"; import { pathToFileURL } from "node:url"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { activityLog, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, +import { activityLog, agents, heartbeatRuns, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, executionWorkspaceRuntimeLeases, executionWorkspaces, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; import { createStorageService } from "../storage/service.js"; import type { StorageService } from "../storage/types.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { migrateWorkspaceTrees } from "./remote-workspace-migration-batch.js"; import { inventoryRemoteWorkspaceMigration, migrateRemoteWorkspace } from "./remote-workspace-migration.js"; import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; @@ -50,7 +51,7 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { await Promise.all(servers.splice(0).map(server => new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())))); vi.unstubAllEnvs(); await db.delete(activityLog); await db.delete(workspaceOperations); await db.delete(executionWorkspaceSnapshots); await db.delete(workspaceRuntimeServices); await db.delete(executionWorkspaceRuntimeLeases); - await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects); + await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects); await db.delete(companies); await fs.rm(root, { recursive: true, force: true }); }); async function seed() { @@ -64,6 +65,142 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { return { companyId, workspaceId, issueId: issue!.id, cwd }; } async function workspace(id: string) { return (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id)))[0]!; } + async function nestedFixture() { + const parent = await seed(), workspaceId = randomUUID(); + const cwd = path.join(parent.cwd, ".paperclip", "worktrees", "child"); + await fs.mkdir(cwd, { recursive: true }); await fs.writeFile(path.join(cwd, "child.txt"), "saved child edits\n"); + const [issue] = await db.insert(issues).values({ companyId: parent.companyId, title: "Nested task" }).returning(); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: parent.companyId, sourceIssueId: issue!.id, + name: "Child workspace", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd, providerRef: cwd }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issue!.id)); + return { parent, child: { companyId: parent.companyId, workspaceId, issueId: issue!.id, cwd } }; + } + it("requires offline maintenance for overlapping physical workspace paths", async () => { + const { parent, child } = await nestedFixture(); + await expect(migrateRemoteWorkspace({ db, storage, ...child, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance"); + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance"); + expect(await fs.readFile(path.join(child.cwd, "child.txt"), "utf8")).toBe("saved child edits\n"); + }); + it("imports nested task files before their parent and retains independently restorable copies", async () => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true }); + await migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true }); + await fs.rm(parent.cwd, { recursive: true }); + for (const [name, fixture] of [["parent", parent], ["child", child]] as const) { + const authority = readRemoteWorkspaceAuthority((await workspace(fixture.workspaceId)).metadata)!; + const ref = await getWorkspaceSnapshot(db, { ...fixture, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, `${name}.pcws`), targetDir = path.join(root, `restored-${name}`); + await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, name === "parent" ? "saved.txt" : "child.txt"), "utf8")) + .toBe(name === "parent" ? "original saved work\n" : "saved child edits\n"); + if (name === "parent") { + expect(await fs.stat(path.join(targetDir, ".paperclip", "worktrees", "child")).catch(() => null)).toBeNull(); + const [snapshot] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, ref!.id)); + expect(snapshot.metadata?.importedWorkspaceChildren).toEqual([expect.objectContaining({ workspaceId: child.workspaceId, + relativePath: ".paperclip/worktrees/child" })]); + } + } + }); + it("requires the offline entry point even when a caller asserts writers stopped", async () => { + const { parent } = await nestedFixture(); + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true })).rejects.toThrow("offline_maintenance_required"); + await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })).rejects.toThrow("offline_maintenance_required"); + }); + it.each(["running", "queued"])("refuses offline archived imports while any execution is %s", async status => { + const f = await seed(); await retainDirectory(f); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Unrelated writer" }).returning(); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, status }); + await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })) + .rejects.toThrow("active_execution_requires_drain"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + }); + it("detects overlapping folders reached through symlinked parent directories", async () => { + const f = await seed(), alias = path.join(root, "alias"); await fs.symlink(root, alias); + await db.insert(executionWorkspaces).values({ companyId: f.companyId, name: "Physical alias", mode: "isolated_workspace", + strategyType: "project_primary", providerType: "local_fs", cwd: path.join(alias, "original") }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance"); + }); + it.each(["current", "legacy"])("keeps the parent capture scope when children migrate between retries (%s)", async journal => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const put = storage.putFile.bind(storage); let fail = true; + storage.putFile = async input => { const saved = await put(input); if (fail) { fail = false; throw new Error("upload reply lost"); } return saved; }; + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true })).rejects.toThrow("upload reply lost"); + if (journal === "legacy") { + const [operation] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.executionWorkspaceId, parent.workspaceId)); + const metadata = { ...operation!.metadata }; delete metadata.nestedWorkspaceScope; + await db.update(workspaceOperations).set({ metadata }).where(eq(workspaceOperations.id, operation!.id)); + } + await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true }); + await migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true }); + const authority = readRemoteWorkspaceAuthority((await workspace(parent.workspaceId)).metadata)!; + const reference = await getWorkspaceSnapshot(db, { ...parent, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, "retried-parent.pcws"), targetDir = path.join(root, "retried-parent"); + await loadWorkspaceSnapshotArchive({ storage, reference: reference!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, ".paperclip/worktrees/child/child.txt"), "utf8")).toBe("saved child edits\n"); + }); + it("blocks distinct legacy records that alias the same folder", async () => { + const f = await seed(), id = randomUUID(); + await db.insert(executionWorkspaces).values({ id, companyId: f.companyId, name: "Historical alias", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: f.cwd }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance"); + }); + it.each(["cleanup_failed", "adapter_managed"])("blocks enrollment instead of abandoning unsupported local files (%s)", async kind => { + const f = await seed(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await db.update(executionWorkspaces).set(kind === "cleanup_failed" ? { status: kind } : { providerType: kind }) + .where(eq(executionWorkspaces.id, f.workspaceId)); + await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })) + .rejects.toThrow(kind === "cleanup_failed" ? "workspace_not_active" : "workspace_provider_unsupported"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n"); + }); + it.each(["changed", "corrupt", "late-change", "abort"])("keeps the parent local when nested coverage is unsafe (%s)", async mode => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true }); + const authority = readRemoteWorkspaceAuthority((await workspace(child.workspaceId)).metadata)!; + const reference = await getWorkspaceSnapshot(db, { ...child, snapshotId: authority.snapshotId! }); + const controller = new AbortController(); + if (mode === "changed") await fs.writeFile(path.join(child.cwd, "child.txt"), "changed child bytes"); + if (mode === "corrupt") await storage.deleteObject(child.companyId, reference!.objectKey); + if (mode === "late-change" || mode === "abort") { + const put = storage.putFile.bind(storage); let once = true; + storage.putFile = async input => { const result = await put(input); if (once) { once = false; + if (mode === "abort") controller.abort(); else await fs.writeFile(path.join(child.cwd, "child.txt"), "changed during upload"); + } return result; }; + } + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true, signal: controller.signal })).rejects.toThrow(); + expect(readRemoteWorkspaceAuthority((await workspace(parent.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(parent.cwd, "saved.txt"), "utf8")).toBe("original saved work\n"); + if (mode === "abort") { + await migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true }); + expect(readRemoteWorkspaceAuthority((await workspace(parent.workspaceId)).metadata)?.snapshotId).toBeTruthy(); + } + }); + it("resumes an interrupted maintenance batch in leaf-first order without duplicating imports", async () => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const seen: string[] = [], controller = new AbortController(); + await expect(migrateWorkspaceTrees(db, { storage, signal: controller.signal, phase: name => { + seen.push(name); if (name === `workspace:${parent.workspaceId}`) controller.abort(); + } })).rejects.toThrow(); + expect(seen).toEqual([`workspace:${child.workspaceId}`, `workspace:${parent.workspaceId}`]); + const childAuthority = readRemoteWorkspaceAuthority((await workspace(child.workspaceId)).metadata); + expect(await migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })).toBe(1); + expect(await migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })).toBe(0); + expect(readRemoteWorkspaceAuthority((await workspace(child.workspaceId)).metadata)).toEqual(childAuthority); + }); + it("checks transitive nested coverage before omitting a child tree", async () => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const id = randomUUID(), cwd = path.join(child.cwd, ".paperclip", "worktrees", "grandchild"); + await fs.mkdir(cwd, { recursive: true }); await fs.writeFile(path.join(cwd, "deep.txt"), "deep work"); + const [issue] = await db.insert(issues).values({ companyId: parent.companyId, title: "Grandchild task" }).returning(); + await db.insert(executionWorkspaces).values({ id, companyId: parent.companyId, sourceIssueId: issue!.id, + name: "Grandchild", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd }); + await migrateRemoteWorkspace({ db, storage, companyId: parent.companyId, workspaceId: id, apply: true, writersStopped: true }); + await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true }); + await fs.writeFile(path.join(cwd, "deep.txt"), "new deep work"); + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true })).rejects.toThrow("nested_workspace_source_changed"); + expect(readRemoteWorkspaceAuthority((await workspace(parent.workspaceId)).metadata)).toBeNull(); + }); async function gitArchiveFixture(repoUrl = "https://example.invalid/repository.git") { const f = await seed(), projectId = randomUUID(), projectWorkspaceId = randomUUID(), branchName = "saved-task"; vi.stubEnv("PAPERCLIP_HOME", root); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "default"); diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index 47dd4c7435f..91cdc407c22 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -17,6 +17,7 @@ import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-rete import { materializeTaskDirectorySnapshot, removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; import { loadTaskDirectoryRestorePoint, mayBeIsolatedTaskDirectoryWorkspace, readTaskDirectoryRetention, validateTaskDirectoryRestorePointRef } from "./isolated-task-directory-retention.js"; import { inspectArchivedGitMigrationSource, materializeArchivedGitMigrationSource, type ArchivedGitMigrationSource } from "./remote-workspace-migration-git.js"; +import { prepareNestedWorkspaceImports, validateImportedWorkspaceChildren, workspaceMigrationLayoutBlockers } from "./remote-workspace-migration-tree.js"; const SCHEMA = "paperclip.remote-workspace-migration.v1"; type Workspace = typeof executionWorkspaces.$inferSelect; @@ -38,7 +39,7 @@ const gitSourceProof = (source: ArchivedGitMigrationSource | null) => source ? { commit: source.commit, sourcePath: source.sourcePath, cleanupOperationId: source.cleanupOperationId, } : null; -async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string }) { +async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string; writersStopped?: boolean }) { const [workspace] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))); if (!workspace) throw new Error("remote_workspace_migration_workspace_missing"); const authority = readRemoteWorkspaceAuthority(workspace.metadata); @@ -77,6 +78,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri ? "restore_point_expired" : "archived_git_source_unavailable"); } } if (!retainedDirectory && !archivedGitSource && (!entry?.isDirectory() || entry.isSymbolicLink())) reasons.push("local_source_unsafe_or_missing"); + if (entry?.isDirectory() && !entry.isSymbolicLink()) reasons.push(...await workspaceMigrationLayoutBlockers(db, workspace, input.writersStopped)); } const [issue] = workspace.sourceIssueId ? await db.select().from(issues).where(and(eq(issues.companyId, workspace.companyId), eq(issues.id, workspace.sourceIssueId))) : []; // Historical records still own their saved files after the task selects a @@ -142,7 +144,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri return { workspace, reasons: [...new Set(reasons)], authority, marker, source, issueBinding: issue?.executionWorkspaceId ?? null, retainedDirectory, archivedGitSource }; } -export async function inventoryRemoteWorkspaceMigration(db: Db, input: { companyId: string; workspaceId: string }) { +export async function inventoryRemoteWorkspaceMigration(db: Db, input: { companyId: string; workspaceId: string; writersStopped?: boolean }) { const result = await inspect(db, input); return { companyId: input.companyId, workspaceId: input.workspaceId, eligible: result.reasons.length === 0, alreadyMigrated: Boolean(result.authority), reasons: result.reasons, localPath: result.workspace.cwd ?? result.workspace.providerRef, @@ -153,6 +155,7 @@ export async function inventoryRemoteWorkspaceMigration(db: Db, input: { company /** Administrative import. The CLI enforces external deployment prerequisites. */ export async function migrateRemoteWorkspace(input: { db: Db; storage: StorageService; companyId: string; workspaceId: string; apply?: boolean; source?: RemoteWorkspaceSource; + writersStopped?: boolean; signal?: AbortSignal; }) { if (!input.apply) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: false }; return withExecutionWorkspaceLifecycleLock(input.db, input.workspaceId, async lock => { @@ -183,6 +186,7 @@ export async function migrateRemoteWorkspace(input: { const [operation] = await tx.insert(workspaceOperations).values({ companyId: workspace.companyId, executionWorkspaceId: workspace.id, issueId: workspace.sourceIssueId, cwd: workspace.cwd ?? workspace.providerRef, phase: "remote_workspace_migration", status: "running", metadata: { schema: SCHEMA, sourceFingerprint, source, sourceIssueBinding: current.issueBinding, + nestedWorkspaceScope: { version: 1, state: "pending" }, ...(current.archivedGitSource ? { archivedGitSource: gitSourceProof(current.archivedGitSource) } : {}), ...(current.retainedDirectory ? { retainedDirectory: { restorePointId: current.retainedDirectory.restorePoint.id, generation: current.retainedDirectory.generation, digest: current.retainedDirectory.restorePoint.digest } } : {}) } }).returning(); @@ -193,27 +197,52 @@ export async function migrateRemoteWorkspace(input: { return { workspace, authority: null, operation: operation!, source, sourceFingerprint, issueBinding: current.issueBinding, retainedDirectory: current.retainedDirectory, archivedGitSource: current.archivedGitSource }; }); if (reserved.authority) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: true, replayed: true }; - const operation = reserved.operation!; + let operation = reserved.operation!; const snapshot = await beginWorkspaceSnapshot(input.db, { companyId: input.companyId, workspaceId: input.workspaceId, operationId: operation.id, generation: 1, epoch: 1 }); const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-migration-")); try { + input.signal?.throwIfAborted(); const retained = reserved.retainedDirectory?.restorePoint; const sourceRoot = retained ? await materializeTaskDirectorySnapshot({ payload: await loadTaskDirectoryRestorePoint({ storage: input.storage, workspace: reserved.workspace, ref: retained }), target: path.join(scratch, "retained-source"), expectedDigest: retained.digest, }) : reserved.archivedGitSource ? await materializeArchivedGitMigrationSource({ db: input.db, storage: input.storage, workspace: reserved.workspace, source: reserved.archivedGitSource, scratch }) : reserved.workspace.cwd ?? reserved.workspace.providerRef!; + const scope = object(operation.metadata?.nestedWorkspaceScope); + if (operation.metadata?.nestedWorkspaceScope !== undefined && (scope.version !== 1 || !["pending", "fixed"].includes(String(scope.state)))) + throw new Error("nested_workspace_scope_invalid"); + // Journals predating scoped captures included every child. Preserve that + // archive identity even if a later batch has already imported the leaves. + const recordedChildren = scope.state === "fixed" ? validateImportedWorkspaceChildren(scope.children) + : scope.state === "pending" ? undefined : []; + const nested = input.writersStopped && !retained && !reserved.archivedGitSource + ? await prepareNestedWorkspaceImports({ ...input, workspace: reserved.workspace, scratch, recordedChildren }) : []; + if (scope.state === "pending") { + const [saved] = await input.db.update(workspaceOperations).set({ metadata: { ...operation.metadata, + nestedWorkspaceScope: { version: 1, state: "fixed", children: nested } }, updatedAt: new Date() }) + .where(and(eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.id, operation.id), + eq(workspaceOperations.status, "running"))).returning(); + if (!saved) throw new Error("remote_workspace_migration_operation_changed"); + operation = saved; + } + const exclude = nested.map(child => child.relativePath); const archivePath = path.join(scratch, "workspace.pcws"); - const captured = await captureRemoteWorkspaceDirectory({ cwd: sourceRoot, archivePath }); + const captured = await captureRemoteWorkspaceDirectory({ cwd: sourceRoot, archivePath, exclude, signal: input.signal }); const ref = await uploadWorkspaceSnapshot({ db: input.db, storage: input.storage, snapshotId: snapshot.id, - companyId: input.companyId, workspaceId: input.workspaceId, archivePath, manifestDigest: captured.manifestDigest }); + companyId: input.companyId, workspaceId: input.workspaceId, archivePath, manifestDigest: captured.manifestDigest, signal: input.signal }); const restoredArchive = path.join(scratch, "verified.pcws"), restored = path.join(scratch, "restored"); - await loadWorkspaceSnapshotArchive({ storage: input.storage, reference: ref, targetPath: restoredArchive }); - await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir: restored, expectedManifestDigest: ref.manifestDigest }); + await loadWorkspaceSnapshotArchive({ storage: input.storage, reference: ref, targetPath: restoredArchive, signal: input.signal }); + await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir: restored, expectedManifestDigest: ref.manifestDigest, signal: input.signal }); const currentArchive = path.join(scratch, "current.pcws"); - const current = await captureRemoteWorkspaceDirectory({ cwd: sourceRoot, archivePath: currentArchive }); + const current = await captureRemoteWorkspaceDirectory({ cwd: sourceRoot, archivePath: currentArchive, exclude, signal: input.signal }); if (current.manifestDigest !== captured.manifestDigest) throw new Error("remote_workspace_migration_source_changed"); + if (nested.length) { + const verifyScratch = path.join(scratch, "verify-nested"); await fs.mkdir(verifyScratch); + const verified = await prepareNestedWorkspaceImports({ ...input, workspace: reserved.workspace, scratch: verifyScratch, recordedChildren: nested }); + if (canonical(verified) !== canonical(nested)) throw new Error("nested_workspace_source_changed"); + } + input.signal?.throwIfAborted(); await lock.execute(sql`select 1`); const publication = await input.db.transaction(async tx => { const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update"); @@ -231,11 +260,21 @@ export async function migrateRemoteWorkspace(input: { const [uploaded] = await tx.select().from(executionWorkspaceSnapshots).where(and(eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id), eq(executionWorkspaceSnapshots.id, ref.id))).for("update"); if (!uploaded || uploaded.status !== "uploaded" || uploaded.sourceOperationId !== operation.id) throw new Error("remote_workspace_migration_snapshot_changed"); + for (const child of [...nested].sort((a, b) => a.snapshotId.localeCompare(b.snapshotId))) { + const [saved] = await tx.select().from(executionWorkspaceSnapshots).where(and( + eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.id, child.snapshotId), + eq(executionWorkspaceSnapshots.executionWorkspaceId, child.workspaceId), + )).for("update"); + if (!saved || !["committed", "superseded"].includes(saved.status) || saved.manifestDigest !== child.manifestDigest + || saved.sourceOperationId !== child.operationId) throw new Error("nested_workspace_snapshot_changed"); + } const authority: RemoteWorkspaceAuthority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: ref.id, source: reserved.source, location: null, writerRunId: null, dirty: false }; const metadata: Record = { ...workspace.metadata, remoteWorkspace: authority }; delete metadata.remoteWorkspaceMigration; await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`); - await tx.update(executionWorkspaceSnapshots).set({ status: "committed", deleteAfter: null, updatedAt: new Date() }).where(eq(executionWorkspaceSnapshots.id, ref.id)); + await tx.update(executionWorkspaceSnapshots).set({ status: "committed", deleteAfter: null, + metadata: { ...uploaded.metadata, ...(nested.length ? { importedWorkspaceChildren: nested } : {}) }, + updatedAt: new Date() }).where(eq(executionWorkspaceSnapshots.id, ref.id)); await tx.update(executionWorkspaces).set({ providerType: "cloud_sandbox", providerRef: null, cwd: `/workspaces/${workspace.id}`, metadata, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)); await tx.update(workspaceOperations).set({ status: "succeeded", finishedAt: new Date(), updatedAt: new Date(), diff --git a/server/src/services/remote-workspace-snapshot-retention.test.ts b/server/src/services/remote-workspace-snapshot-retention.test.ts index d008d393d38..079d5a0206f 100644 --- a/server/src/services/remote-workspace-snapshot-retention.test.ts +++ b/server/src/services/remote-workspace-snapshot-retention.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { createServer } from "node:http"; import type { AddressInfo } from "node:net"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, executionWorkspaces, issueRecoveryActions, issues, workspaceOperations } from "@paperclipai/db"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; @@ -131,6 +131,41 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () => for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(false); expect(await owner.nextDeadlineAt()).toBeNull(); }); + it("uses the dependency index when a company has many unrelated snapshots", async () => { + const child = await seed(); + await db.execute(sql`insert into execution_workspace_snapshots + (company_id, execution_workspace_id, source_operation_id, generation, epoch, status) + select ${child.companyId}::uuid, gen_random_uuid(), gen_random_uuid(), 1, 1, 'committed' + from generate_series(1, 2500)`); + await db.insert(executionWorkspaceSnapshots).values({ companyId: child.companyId, executionWorkspaceId: randomUUID(), + sourceOperationId: randomUUID(), generation: 1, epoch: 1, status: "committed", + metadata: { importedWorkspaceChildren: [{ workspaceId: child.workspaceId, snapshotId: child.row.id }] } }); + await db.execute(sql`analyze execution_workspace_snapshots`); + const plan = await db.execute(sql`explain (format json) select id from execution_workspace_snapshots + where company_id = ${child.companyId}::uuid and status <> 'deleted' + and metadata ? 'importedWorkspaceChildren' + and metadata->'importedWorkspaceChildren' @> ${JSON.stringify([{ workspaceId: child.workspaceId, snapshotId: child.row.id }])}::jsonb + limit 1`); + expect(JSON.stringify(plan)).toContain("execution_workspace_snapshots_imported_children_idx"); + }); + it("pins imported child bytes until the covering parent snapshot is deleted", async () => { + const child = await seed(), parentId = randomUUID(), workspaceId = randomUUID(); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: child.companyId, name: "Imported parent", mode: "isolated_workspace", strategyType: "project_primary", + metadata: { remoteWorkspace: { snapshotId: parentId } } }); + await db.insert(executionWorkspaceSnapshots).values({ id: parentId, companyId: child.companyId, executionWorkspaceId: workspaceId, + sourceOperationId: randomUUID(), generation: 1, epoch: 1, status: "committed", + metadata: { importedWorkspaceChildren: [{ workspaceId: child.workspaceId, snapshotId: child.row.id }] } }); + const owner = service(); + expect(await owner.tick()).toMatchObject({ pinned: 1, deleted: 0 }); + expect(await owner.nextDeadlineAt()).toBeNull(); + expect((await storage.headObject(child.companyId, child.keys[0]!)).exists).toBe(true); + await db.update(executionWorkspaces).set({ metadata: {} }).where(eq(executionWorkspaces.id, workspaceId)); + await db.update(executionWorkspaceSnapshots).set({ status: "superseded", deleteAfter: new Date(now.getTime() - 1) }) + .where(eq(executionWorkspaceSnapshots.id, parentId)); + await owner.tick(); await owner.tick(); + expect((await storage.headObject(child.companyId, child.keys[0]!)).exists).toBe(false); + expect(await owner.nextDeadlineAt()).toBeNull(); + }); it("purges exact-key S3 versions and delete markers before settling retention", async () => { const s3 = await versionedS3Fixture(); try { diff --git a/server/src/services/remote-workspace-snapshot-retention.ts b/server/src/services/remote-workspace-snapshot-retention.ts index 22977698304..747339eff06 100644 --- a/server/src/services/remote-workspace-snapshot-retention.ts +++ b/server/src/services/remote-workspace-snapshot-retention.ts @@ -23,6 +23,14 @@ async function pinned(db: ReadDb, row: Snapshot) { eq(executionWorkspaces.companyId, row.companyId), eq(executionWorkspaces.id, row.executionWorkspaceId), )); if ((workspace?.metadata?.remoteWorkspace as { snapshotId?: unknown } | undefined)?.snapshotId === row.id) return true; + const [importedBy] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and( + eq(executionWorkspaceSnapshots.companyId, row.companyId), ne(executionWorkspaceSnapshots.status, "deleted"), + sql`${executionWorkspaceSnapshots.metadata} ? 'importedWorkspaceChildren'`, + sql`${executionWorkspaceSnapshots.metadata}->'importedWorkspaceChildren' @> ${JSON.stringify([ + { workspaceId: row.executionWorkspaceId, snapshotId: row.id }, + ])}::jsonb`, + )).limit(1); + if (importedBy) return true; const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( eq(workspaceOperations.companyId, row.companyId), eq(workspaceOperations.executionWorkspaceId, row.executionWorkspaceId), or(inArray(workspaceOperations.status, ["running", "pending"]), @@ -138,7 +146,8 @@ export function remoteWorkspaceSnapshotRetentionService(db: Db, options: { .where(and(eq(executionWorkspaceSnapshots.id, claimed.id), eq(executionWorkspaceSnapshots.status, "delete_requested"))); // A parent inspected earlier in this page can now lose its last pin. // Revisit it once instead of waiting for an unrelated execution event. - if (claimed.parentSnapshotId) rescanRequested = true; + if (claimed.parentSnapshotId || Array.isArray(claimed.metadata?.importedWorkspaceChildren) + && claimed.metadata.importedWorkspaceChildren.length > 0) rescanRequested = true; result.deleted++; } catch (error) { if (signal?.aborted) throw error;