Index imported workspace dependencies and keep migration fail-closed

packages/db/src/migrations/0427_ordinary_xavin.sql

@@ -0,0 +1 @@
+CREATE INDEX "execution_workspace_snapshots_imported_children_idx" ON "execution_workspace_snapshots" USING gin (("metadata"->'importedWorkspaceChildren') jsonb_path_ops) WHERE "execution_workspace_snapshots"."metadata" ? 'importedWorkspaceChildren';

packages/db/src/migrations/meta/0427_snapshot.json

@@ -1,6 +1,6 @@
 {
-  "id": "deb3f1b9-6eb7-447f-adf2-dee69f943c59",
-  "prevId": "8f75a1bb-32d9-4aed-9f93-134a6ae71f79",
+  "id": "dd860bff-55de-4849-8787-58b9de729258",
+  "prevId": "a1fcb4c6-40e7-47a6-b3f2-c43cd9bf332b",
   "version": "7",
   "dialect": "postgresql",
   "tables": {
@@ -2233,6 +2233,74 @@
       "checkConstraints": {},
       "isRLSEnabled": false
     },
+    "public.agent_file_roots": {
+      "name": "agent_file_roots",
+      "schema": "",
+      "columns": {
+        "company_id": {
+          "name": "company_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "agent_id": {
+          "name": "agent_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "generation": {
+          "name": "generation",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "entries": {
+          "name": "entries",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "agent_file_roots_company_id_agent_id_agents_company_id_id_fk": {
+          "name": "agent_file_roots_company_id_agent_id_agents_company_id_id_fk",
+          "tableFrom": "agent_file_roots",
+          "tableTo": "agents",
+          "columnsFrom": [
+            "company_id",
+            "agent_id"
+          ],
+          "columnsTo": [
+            "company_id",
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "agent_file_roots_company_id_agent_id_pk": {
+          "name": "agent_file_roots_company_id_agent_id_pk",
+          "columns": [
+            "company_id",
+            "agent_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
     "public.agent_identity_keys": {
       "name": "agent_identity_keys",
       "schema": "",
@@ -28757,6 +28825,327 @@
       "checkConstraints": {},
       "isRLSEnabled": false
     },
+    "public.execution_admission_project_charges": {
+      "name": "execution_admission_project_charges",
+      "schema": "",
+      "columns": {
+        "admission_id": {
+          "name": "admission_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "company_id": {
+          "name": "company_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "project_id": {
+          "name": "project_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {
+        "execution_admission_project_charges_project_idx": {
+          "name": "execution_admission_project_charges_project_idx",
+          "columns": [
+            {
+              "expression": "company_id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "project_id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        }
+      },
+      "foreignKeys": {
+        "execution_admission_project_charges_company_admission_fk": {
+          "name": "execution_admission_project_charges_company_admission_fk",
+          "tableFrom": "execution_admission_project_charges",
+          "tableTo": "execution_admissions",
+          "columnsFrom": [
+            "company_id",
+            "admission_id"
+          ],
+          "columnsTo": [
+            "company_id",
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "execution_admission_project_charges_company_project_fk": {
+          "name": "execution_admission_project_charges_company_project_fk",
+          "tableFrom": "execution_admission_project_charges",
+          "tableTo": "projects",
+          "columnsFrom": [
+            "company_id",
+            "project_id"
+          ],
+          "columnsTo": [
+            "company_id",
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "execution_admission_project_charges_admission_id_project_id_pk": {
+          "name": "execution_admission_project_charges_admission_id_project_id_pk",
+          "columns": [
+            "admission_id",
+            "project_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.execution_admissions": {
+      "name": "execution_admissions",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "company_id": {
+          "name": "company_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "run_id": {
+          "name": "run_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attempt": {
+          "name": "attempt",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "primary_project_id": {
+          "name": "primary_project_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "context_revision": {
+          "name": "context_revision",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "settlement_id": {
+          "name": "settlement_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "settled_at": {
+          "name": "settled_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {
+        "execution_admissions_company_id_uq": {
+          "name": "execution_admissions_company_id_uq",
+          "columns": [
+            {
+              "expression": "company_id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": true,
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        },
+        "execution_admissions_run_attempt_uq": {
+          "name": "execution_admissions_run_attempt_uq",
+          "columns": [
+            {
+              "expression": "run_id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "attempt",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": true,
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        },
+        "execution_admissions_live_run_uq": {
+          "name": "execution_admissions_live_run_uq",
+          "columns": [
+            {
+              "expression": "run_id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": true,
+          "where": "\"execution_admissions\".\"settled_at\" is null",
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        }
+      },
+      "foreignKeys": {
+        "execution_admissions_company_id_companies_id_fk": {
+          "name": "execution_admissions_company_id_companies_id_fk",
+          "tableFrom": "execution_admissions",
+          "tableTo": "companies",
+          "columnsFrom": [
+            "company_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        },
+        "execution_admissions_run_id_heartbeat_runs_id_fk": {
+          "name": "execution_admissions_run_id_heartbeat_runs_id_fk",
+          "tableFrom": "execution_admissions",
+          "tableTo": "heartbeat_runs",
+          "columnsFrom": [
+            "run_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {
+        "execution_admissions_attempt_check": {
+          "name": "execution_admissions_attempt_check",
+          "value": "\"execution_admissions\".\"attempt\" > 0"
+        },
+        "execution_admissions_settlement_check": {
+          "name": "execution_admissions_settlement_check",
+          "value": "(\"execution_admissions\".\"settlement_id\" is null) = (\"execution_admissions\".\"settled_at\" is null)"
+        }
+      },
+      "isRLSEnabled": false
+    },
+    "public.project_task_execution_policies": {
+      "name": "project_task_execution_policies",
+      "schema": "",
+      "columns": {
+        "project_id": {
+          "name": "project_id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "company_id": {
+          "name": "company_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "max_concurrent_tasks": {
+          "name": "max_concurrent_tasks",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "project_task_execution_policies_company_project_fk": {
+          "name": "project_task_execution_policies_company_project_fk",
+          "tableFrom": "project_task_execution_policies",
+          "tableTo": "projects",
+          "columnsFrom": [
+            "company_id",
+            "project_id"
+          ],
+          "columnsTo": [
+            "company_id",
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {
+        "project_task_execution_policies_maximum_check": {
+          "name": "project_task_execution_policies_maximum_check",
+          "value": "\"project_task_execution_policies\".\"max_concurrent_tasks\" is null or \"project_task_execution_policies\".\"max_concurrent_tasks\" > 0"
+        }
+      },
+      "isRLSEnabled": false
+    },
     "public.execution_workspace_repositories": {
       "name": "execution_workspace_repositories",
       "schema": "",
@@ -29931,6 +30320,22 @@
           "concurrently": false,
           "method": "btree",
           "with": {}
+        },
+        "execution_workspace_snapshots_imported_children_idx": {
+          "name": "execution_workspace_snapshots_imported_children_idx",
+          "columns": [
+            {
+              "expression": "(\"metadata\"->'importedWorkspaceChildren') jsonb_path_ops",
+              "asc": true,
+              "isExpression": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "where": "\"execution_workspace_snapshots\".\"metadata\" ? 'importedWorkspaceChildren'",
+          "concurrently": false,
+          "method": "gin",
+          "with": {}
         }
       },
       "foreignKeys": {},
@@ -50412,6 +50817,12 @@
           "primaryKey": false,
           "notNull": false
         },
+        "created_by_run_id": {
+          "name": "created_by_run_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
         "dispatch_fingerprint": {
           "name": "dispatch_fingerprint",
           "type": "text",
@@ -62374,6 +62785,12 @@
           "primaryKey": false,
           "notNull": false
         },
+        "cleanup_verified_at": {
+          "name": "cleanup_verified_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
         "created_at": {
           "name": "created_at",
           "type": "timestamp with time zone",

packages/db/src/migrations/meta/_journal.json

@@ -2976,6 +2976,13 @@
       "when": 1793733446322,
       "tag": "0426_absent_guardian",
       "breakpoints": true
+    },
+    {
+      "idx": 427,
+      "version": "7",
+      "when": 1793736361959,
+      "tag": "0427_ordinary_xavin",
+      "breakpoints": true
     }
   ]
 }

packages/db/src/schema/execution_workspace_snapshots.ts

@@ -1,4 +1,5 @@
 import { bigint, index, integer, jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core";
+import { sql } from "drizzle-orm";
 
 /** Durable object ledger deliberately survives deletion of its source records. */
 export const executionWorkspaceSnapshots = pgTable("execution_workspace_snapshots", {
@@ -27,4 +28,7 @@ export const executionWorkspaceSnapshots = pgTable("execution_workspace_snapshot
   generationUniqueIdx: uniqueIndex("execution_workspace_snapshots_generation_uniq").on(table.companyId, table.executionWorkspaceId, table.generation),
   workspaceIdx: index("execution_workspace_snapshots_workspace_idx").on(table.companyId, table.executionWorkspaceId),
   cleanupIdx: index("execution_workspace_snapshots_cleanup_idx").on(table.status, table.deleteAfter),
+  importedChildrenIdx: index("execution_workspace_snapshots_imported_children_idx")
+    .using("gin", sql`(${table.metadata}->'importedWorkspaceChildren') jsonb_path_ops`)
+    .where(sql`${table.metadata} ? 'importedWorkspaceChildren'`),
 }));

server/src/services/remote-workspace-migration-batch.ts

@@ -15,6 +15,9 @@ export async function migrateWorkspaceTrees(db: Db, input: {
   let migrated = 0;
   for (;;) {
     input.signal.throwIfAborted();
+    // Match the final ephemeral-storage contract, not just the importer's
+    // supported subset. Silently skipping an unsupported provider or unsettled
+    // lifecycle state would abandon files when the original volume is detached.
     const rows = await db.select({ companyId: executionWorkspaces.companyId, workspaceId: executionWorkspaces.id })
       .from(executionWorkspaces).where(and(isNotNull(executionWorkspaces.cwd),
         sql`${executionWorkspaces.providerType} <> 'plugin'`,

server/src/services/remote-workspace-migration.test.ts

@@ -145,6 +145,15 @@ describe.skipIf(!support.supported)("remote workspace migration", () => {
     await db.insert(executionWorkspaces).values({ id, companyId: f.companyId, name: "Historical alias", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: f.cwd });
     await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance");
   });
+  it.each(["cleanup_failed", "adapter_managed"])("blocks enrollment instead of abandoning unsupported local files (%s)", async kind => {
+    const f = await seed(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
+    await db.update(executionWorkspaces).set(kind === "cleanup_failed" ? { status: kind } : { providerType: kind })
+      .where(eq(executionWorkspaces.id, f.workspaceId));
+    await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} }))
+      .rejects.toThrow(kind === "cleanup_failed" ? "workspace_not_active" : "workspace_provider_unsupported");
+    expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
+    expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
+  });
   it.each(["changed", "corrupt", "late-change", "abort"])("keeps the parent local when nested coverage is unsafe (%s)", async mode => {
     const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
     await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true });

server/src/services/remote-workspace-snapshot-retention.test.ts

@@ -5,7 +5,7 @@ import path from "node:path";
 import { createServer } from "node:http";
 import type { AddressInfo } from "node:net";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
-import { eq } from "drizzle-orm";
+import { eq, sql } from "drizzle-orm";
 import { applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
   executionWorkspaces, issueRecoveryActions, issues, workspaceOperations } from "@paperclipai/db";
 import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
@@ -131,6 +131,23 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () =>
     for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(false);
     expect(await owner.nextDeadlineAt()).toBeNull();
   });
+  it("uses the dependency index when a company has many unrelated snapshots", async () => {
+    const child = await seed();
+    await db.execute(sql`insert into execution_workspace_snapshots
+      (company_id, execution_workspace_id, source_operation_id, generation, epoch, status)
+      select ${child.companyId}::uuid, gen_random_uuid(), gen_random_uuid(), 1, 1, 'committed'
+      from generate_series(1, 2500)`);
+    await db.insert(executionWorkspaceSnapshots).values({ companyId: child.companyId, executionWorkspaceId: randomUUID(),
+      sourceOperationId: randomUUID(), generation: 1, epoch: 1, status: "committed",
+      metadata: { importedWorkspaceChildren: [{ workspaceId: child.workspaceId, snapshotId: child.row.id }] } });
+    await db.execute(sql`analyze execution_workspace_snapshots`);
+    const plan = await db.execute(sql`explain (format json) select id from execution_workspace_snapshots
+      where company_id = ${child.companyId}::uuid and status <> 'deleted'
+        and metadata ? 'importedWorkspaceChildren'
+        and metadata->'importedWorkspaceChildren' @> ${JSON.stringify([{ workspaceId: child.workspaceId, snapshotId: child.row.id }])}::jsonb
+      limit 1`);
+    expect(JSON.stringify(plan)).toContain("execution_workspace_snapshots_imported_children_idx");
+  });
   it("pins imported child bytes until the covering parent snapshot is deleted", async () => {
     const child = await seed(), parentId = randomUUID(), workspaceId = randomUUID();
     await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: child.companyId, name: "Imported parent", mode: "isolated_workspace", strategyType: "project_primary",

server/src/services/remote-workspace-snapshot-retention.ts

@@ -25,6 +25,7 @@ async function pinned(db: ReadDb, row: Snapshot) {
   if ((workspace?.metadata?.remoteWorkspace as { snapshotId?: unknown } | undefined)?.snapshotId === row.id) return true;
   const [importedBy] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
     eq(executionWorkspaceSnapshots.companyId, row.companyId), ne(executionWorkspaceSnapshots.status, "deleted"),
+    sql`${executionWorkspaceSnapshots.metadata} ? 'importedWorkspaceChildren'`,
     sql`${executionWorkspaceSnapshots.metadata}->'importedWorkspaceChildren' @> ${JSON.stringify([
       { workspaceId: row.executionWorkspaceId, snapshotId: row.id },
     ])}::jsonb`,