diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 09971d890b4..332d1836782 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -2405,3 +2405,22 @@ but those values no longer select the active workspace. API reads derive retain the existing account and company-audit checks. Replaying the migration preserves explicit Classic choices. Agent Chat ignores its retired stored flag without rewriting instance settings. No task, agent, or execution state changes. + +## Project environment rolling compatibility + +New projects select repository storage rather than a persistent run environment. +The nullable `projects.environment_id` column remains a database compatibility +field for older servers; the current project API omits it and does not use it +for placement. + +When migrations `0466_drop_project_environment.sql` and +`0467_fair_human_torch.sql` are both pending, the canonical migration runner +qualifies their exact SQL hashes and applies them in one transaction. It locks +`projects`, saves every nonnull legacy environment value, executes both published +migrations and records both genuine ledger entries, then restores those values +before commit. Old readers and writers wait for the table lock and continue +against the retained column. Any failure rolls back the DDL, values and receipts. +If `0466` was already committed, `0467` adds the nullable column and its original +`ON DELETE SET NULL` foreign key normally; it cannot recover values previously +removed by that committed migration. Deployments that pre-stage this pair must +use this migration runner rather than execute the SQL files independently. diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index 4b574436ef5..58373c7c9e7 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -11,6 +11,12 @@ import * as schema from "./schema/index.js"; const MIGRATIONS_FOLDER = fileURLToPath(new URL("./migrations", import.meta.url)); const DRIZZLE_MIGRATIONS_TABLE = "__drizzle_migrations"; const MIGRATIONS_JOURNAL_JSON = fileURLToPath(new URL("./migrations/meta/_journal.json", import.meta.url)); +const PROJECT_ENVIRONMENT_OVERLAP = { + dropFile: "0466_drop_project_environment.sql", + dropSha256: "07ca1c666c6471fdb9ce7a0a5fb31bf1a074138466fa8daae50b53199b8e0d7c", + repairFile: "0467_fair_human_torch.sql", + repairSha256: "50ce0eb2567570b098bea1de92fe6a220d139e4ba83438195fbbad1be51e31a1", +} as const; function createUtilitySql(url: string) { return postgres(url, { max: 1, onnotice: () => {} }); @@ -706,7 +712,29 @@ async function applyPendingMigrationsManually( folderMillisByFileName.get(migrationFile) ?? Date.now(), ); }; - if (MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS.has(migrationFile)) { + if (migrationFile === PROJECT_ENVIRONMENT_OVERLAP.dropFile) { + const repairFile = PROJECT_ENVIRONMENT_OVERLAP.repairFile; + const repairContent = await readMigrationFileContent(repairFile); + const repairHash = createHash("sha256").update(repairContent).digest("hex"); + const dropIndex = orderedPendingMigrations.indexOf(migrationFile); + if (hash !== PROJECT_ENVIRONMENT_OVERLAP.dropSha256 + || repairHash !== PROJECT_ENVIRONMENT_OVERLAP.repairSha256 + || orderedPendingMigrations[dropIndex + 1] !== repairFile + || await migrationHistoryEntryExists(sql, qualifiedTable, columnNames, repairFile, repairHash)) { + throw new Error("Project environment migration pair does not match the qualified overlap repair"); + } + await runInTransaction(sql, async () => { + await sql.unsafe('LOCK TABLE "public"."projects" IN ACCESS EXCLUSIVE MODE'); + await sql.unsafe(`CREATE TEMP TABLE paperclip_project_environment_overlap ON COMMIT DROP AS + SELECT id, environment_id FROM "public"."projects" WHERE environment_id IS NOT NULL`); + await applyMigration(); + for (const statement of splitMigrationStatements(repairContent)) await sql.unsafe(statement); + await sql.unsafe(`UPDATE "public"."projects" AS project SET environment_id = legacy.environment_id + FROM pg_temp.paperclip_project_environment_overlap AS legacy WHERE project.id = legacy.id`); + await recordMigrationHistoryEntry(sql, qualifiedTable, columnNames, repairFile, repairHash, + folderMillisByFileName.get(repairFile) ?? Date.now()); + }); + } else if (MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS.has(migrationFile)) { await applyMigration(); } else { await runInTransaction(sql, applyMigration); diff --git a/packages/db/src/migrations/0467_fair_human_torch.sql b/packages/db/src/migrations/0467_fair_human_torch.sql new file mode 100644 index 00000000000..69547fc747a --- /dev/null +++ b/packages/db/src/migrations/0467_fair_human_torch.sql @@ -0,0 +1,2 @@ +ALTER TABLE "projects" ADD COLUMN "environment_id" uuid;--> statement-breakpoint +ALTER TABLE "projects" ADD CONSTRAINT "projects_environment_id_environments_id_fk" FOREIGN KEY ("environment_id") REFERENCES "public"."environments"("id") ON DELETE set null ON UPDATE no action; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0462_snapshot.json b/packages/db/src/migrations/meta/0467_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0462_snapshot.json rename to packages/db/src/migrations/meta/0467_snapshot.json index e374e515147..3167bbd1e50 100644 --- a/packages/db/src/migrations/meta/0462_snapshot.json +++ b/packages/db/src/migrations/meta/0467_snapshot.json @@ -1,6 +1,6 @@ { - "id": "0287d884-1fee-4e8d-9ae0-2be0aac45b71", - "prevId": "f409684f-c629-4648-9123-a5e1064308ef", + "id": "426d1f42-0983-4c6b-bb8d-5c798eee915a", + "prevId": "593f81c4-2b4d-4fb4-a3e3-848b19bc9b0f", "version": "7", "dialect": "postgresql", "tables": { @@ -6233,7 +6233,7 @@ "name": "issuer", "type": "text", "primaryKey": false, - "notNull": true + "notNull": false }, "account_id": { "name": "account_id", @@ -6309,11 +6309,11 @@ } }, "indexes": { - "account_issuer_account_id_uq": { - "name": "account_issuer_account_id_uq", + "account_provider_account_id_uq": { + "name": "account_provider_account_id_uq", "columns": [ { - "expression": "issuer", + "expression": "provider_id", "isExpression": false, "asc": true, "nulls": "last" @@ -51034,6 +51034,12 @@ "primaryKey": false, "notNull": false }, + "environment_id": { + "name": "environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, "target_date": { "name": "target_date", "type": "date", @@ -51209,6 +51215,19 @@ "onDelete": "no action", "onUpdate": "no action" }, + "projects_environment_id_environments_id_fk": { + "name": "projects_environment_id_environments_id_fk", + "tableFrom": "projects", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, "projects_git_authorization_grant_id_connection_grants_id_fk": { "name": "projects_git_authorization_grant_id_connection_grants_id_fk", "tableFrom": "projects", @@ -59603,6 +59622,13 @@ "notNull": true, "default": "''" }, + "pending_reveal": { + "name": "pending_reveal", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, "allowed_actions": { "name": "allowed_actions", "type": "jsonb", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 93aad29fed9..80dda74cfd5 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3256,6 +3256,13 @@ "when": 1794408985337, "tag": "0466_drop_project_environment", "breakpoints": true + }, + { + "idx": 467, + "version": "7", + "when": 1794414797116, + "tag": "0467_fair_human_torch", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/project-environment-overlap-migration.test.ts b/packages/db/src/project-environment-overlap-migration.test.ts new file mode 100644 index 00000000000..1ffad040217 --- /dev/null +++ b/packages/db/src/project-environment-overlap-migration.test.ts @@ -0,0 +1,130 @@ +import { createHash, randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import postgres from "postgres"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { applyPendingMigrations, inspectMigrations } from "./client.js"; +import { + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./test-embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; +const migrations = ["0466_drop_project_environment.sql", "0467_fair_human_torch.sql"]; +const hashes = await Promise.all(migrations.map(async (file) => createHash("sha256") + .update(await readFile(new URL(`./migrations/${file}`, import.meta.url))).digest("hex"))); + +describePostgres("project environment rolling migration", () => { + let database: Awaited>; + let sql: ReturnType; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-project-environment-overlap-"); + sql = postgres(database.connectionString, { max: 1, onnotice: () => {} }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + afterAll(async () => { + await sql?.end(); + await database?.cleanup(); + }); + + async function prepare() { + await sql`DELETE FROM projects`; + await sql`DELETE FROM drizzle.__drizzle_migrations WHERE hash IN ${sql(hashes)}`; + const environmentIds = [randomUUID(), randomUUID()]; + await sql`INSERT INTO environments (id, name, driver) VALUES + (${environmentIds[0]}, ${randomUUID()}, 'ssh'), (${environmentIds[1]}, ${randomUUID()}, 'ssh')`; + const projectIds = [randomUUID(), randomUUID(), randomUUID()]; + for (let index = 0; index < projectIds.length; index++) { + const companyId = randomUUID(); + await sql`INSERT INTO companies (id, name, issue_prefix) VALUES (${companyId}, 'Legacy company', ${`P${index}${companyId.slice(0, 6)}`})`; + await sql`INSERT INTO projects (id, company_id, name, environment_id) VALUES + (${projectIds[index]}, ${companyId}, 'Legacy project', ${environmentIds[index] ?? null})`; + } + return { environmentIds, projectIds }; + } + + const rows = () => sql`SELECT id, environment_id FROM projects ORDER BY id`; + const receipts = () => sql`SELECT hash FROM drizzle.__drizzle_migrations WHERE hash IN ${sql(hashes)} ORDER BY hash`; + + it("preserves committed old-writer values and gives concurrent old readers an unbroken column", async () => { + const { environmentIds, projectIds } = await prepare(); + const writer = postgres(database.connectionString, { max: 1, onnotice: () => {} }); + const readerUrl = new URL(database.connectionString); + readerUrl.searchParams.set("application_name", "project-column-overlap-reader"); + const reader = postgres(readerUrl.toString(), { max: 1, onnotice: () => {} }); + const migrationUrl = new URL(database.connectionString); + migrationUrl.searchParams.set("application_name", "project-column-overlap-migrator"); + let migration: Promise | undefined; + let read: Promise | undefined; + try { + await writer`BEGIN`; + await writer`UPDATE projects SET environment_id = ${environmentIds[1]} WHERE id = ${projectIds[0]}`; + migration = applyPendingMigrations(migrationUrl.toString()); + await expect.poll(async () => (await sql`SELECT EXISTS ( + SELECT 1 FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid + WHERE l.relation = 'projects'::regclass AND l.mode = 'AccessExclusiveLock' AND NOT l.granted + AND a.application_name = 'project-column-overlap-migrator') AS waiting`)[0]?.waiting, + { timeout: 10_000 }).toBe(true); + read = reader`SELECT id, environment_id FROM projects ORDER BY id`.then((result) => Array.from(result)); + await expect.poll(async () => (await sql`SELECT EXISTS ( + SELECT 1 FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid + WHERE l.relation = 'projects'::regclass AND l.mode = 'AccessShareLock' AND NOT l.granted + AND a.application_name = 'project-column-overlap-reader') AS waiting`)[0]?.waiting, + { timeout: 10_000 }).toBe(true); + const before = Array.from(await writer`SELECT id, environment_id FROM projects ORDER BY id`); + await writer`COMMIT`; + await migration; + expect(await read).toEqual(before); + expect(Array.from(await rows())).toEqual(before); + expect(Array.from(await receipts())).toEqual(hashes.toSorted().map((hash) => ({ hash }))); + const state = await inspectMigrations(database.connectionString); + expect(state.status).toBe("upToDate"); + await applyPendingMigrations(database.connectionString); + expect(Array.from(await rows())).toEqual(before); + expect(await receipts()).toHaveLength(2); + await writer`UPDATE projects SET environment_id = ${environmentIds[0]} WHERE id = ${projectIds[0]}`; + await sql`DELETE FROM environments WHERE id = ${environmentIds[0]}`; + expect((await sql`SELECT environment_id FROM projects WHERE id = ${projectIds[0]}`)[0]?.environment_id).toBeNull(); + } finally { + await writer`ROLLBACK`; + await Promise.allSettled([migration, read]); + await Promise.all([writer.end(), reader.end()]); + } + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + it("rolls back both DDL and genuine receipts if restoring the legacy values fails", async () => { + await prepare(); + const before = Array.from(await rows()); + await sql.unsafe(`CREATE FUNCTION reject_project_environment_restore() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'project_environment_restore_fixture'; END $$`); + await sql.unsafe(`CREATE TRIGGER reject_project_environment_restore BEFORE UPDATE ON projects + FOR EACH ROW EXECUTE FUNCTION reject_project_environment_restore()`); + try { + await expect(applyPendingMigrations(database.connectionString)).rejects.toThrow("project_environment_restore_fixture"); + expect(Array.from(await rows())).toEqual(before); + expect(await receipts()).toHaveLength(0); + } finally { + await sql.unsafe("DROP TRIGGER reject_project_environment_restore ON projects"); + await sql.unsafe("DROP FUNCTION reject_project_environment_restore()"); + } + await applyPendingMigrations(database.connectionString); + expect(Array.from(await rows())).toEqual(before); + expect(await receipts()).toHaveLength(2); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + it("applies only the additive repair when the published drop was already committed", async () => { + await sql`DELETE FROM projects`; + await sql.unsafe('ALTER TABLE projects DROP COLUMN environment_id'); + await sql`DELETE FROM drizzle.__drizzle_migrations WHERE hash = ${hashes[1]}`; + await applyPendingMigrations(database.connectionString); + expect((await sql`SELECT data_type, is_nullable FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'projects' AND column_name = 'environment_id'`)[0]) + .toEqual({ data_type: "uuid", is_nullable: "YES" }); + expect((await sql`SELECT confdeltype FROM pg_constraint WHERE conrelid = 'projects'::regclass + AND conname = 'projects_environment_id_environments_id_fk'`)[0]?.confdeltype).toBe("n"); + expect(await receipts()).toHaveLength(2); + expect((await inspectMigrations(database.connectionString)).status).toBe("upToDate"); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); +}); diff --git a/packages/db/src/schema/projects.ts b/packages/db/src/schema/projects.ts index 10e320891bb..9a4fe69e521 100644 --- a/packages/db/src/schema/projects.ts +++ b/packages/db/src/schema/projects.ts @@ -4,6 +4,7 @@ import type { AgentEnvConfig } from "@paperclipai/shared"; import { companies } from "./companies.js"; import { goals } from "./goals.js"; import { agents } from "./agents.js"; +import { environments } from "./environments.js"; import { connectionGrants } from "./tool_access.js"; export const projects = pgTable( @@ -19,6 +20,7 @@ export const projects = pgTable( personalOwnerUserId: text("personal_owner_user_id"), status: text("status").notNull().default("backlog"), leadAgentId: uuid("lead_agent_id").references(() => agents.id), + legacyEnvironmentId: uuid("environment_id").references(() => environments.id, { onDelete: "set null" }), targetDate: date("target_date"), color: text("color"), icon: text("icon"), diff --git a/server/src/__tests__/project-environments.test.ts b/server/src/__tests__/project-environments.test.ts index 0e66fb9bb1c..53680272e43 100644 --- a/server/src/__tests__/project-environments.test.ts +++ b/server/src/__tests__/project-environments.test.ts @@ -110,6 +110,7 @@ describe.skipIf(!support.supported)("project environment compatibility", () => { const core = await svc.create(f.companyId, { name: "Core project" }, { environmentId: f.ssh.id, expectedStorageProviderPluginId: null }); const managed = await svc.create(f.companyId, { name: "Managed project" }, { environmentId: f.computer.id }); expect(core).not.toHaveProperty("environmentId"); + expect(core).not.toHaveProperty("legacyEnvironmentId"); expect(core.workspaceProvider?.pluginId).toBeNull(); expect(managed.workspaceProvider?.pluginId).toBe(f.pluginId); await expect(svc.update(core.id, { executionWorkspacePolicy: { enabled: true, storageProviderPluginId: f.pluginId } })) diff --git a/server/src/__tests__/projects-list-archived-routes.test.ts b/server/src/__tests__/projects-list-archived-routes.test.ts index 9e6db4608d8..2ad0bf8397c 100644 --- a/server/src/__tests__/projects-list-archived-routes.test.ts +++ b/server/src/__tests__/projects-list-archived-routes.test.ts @@ -1,8 +1,9 @@ import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; +import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { companies, companyMemberships, createDb, projectAccessMembers, projects } from "@paperclipai/db"; +import { activityLog, companies, companyMemberships, createDb, environments, projectAccessMembers, projects } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -54,6 +55,7 @@ describeEmbeddedPostgres("project list archived route defaults", () => { }, 120_000); afterEach(async () => { + await db.delete(activityLog); await db.delete(companyMemberships); await db.delete(projects); await db.delete(companies); @@ -108,6 +110,20 @@ describeEmbeddedPostgres("project list archived route defaults", () => { expect(res.status).toBe(200); expect(res.body.map((project: { id: string }) => project.id).sort()).toEqual([activeProjectId, archivedProjectId].sort()); }); + + it("omits the database-only legacy environment from the DELETE response", async () => { + const { activeProjectId, companyId } = await seed(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "user-1", + status: "active", membershipRole: "operator" }); + const [environment] = await db.insert(environments).values({ name: randomUUID(), driver: "ssh" }).returning(); + await db.update(projects).set({ legacyEnvironmentId: environment.id }).where(eq(projects.id, activeProjectId)); + const app = createApp(db, boardActor(companyId)); + const res = await request(app).delete(`/api/projects/${activeProjectId}`); + expect(res.status).toBe(200); + expect(res.body.id).toBe(activeProjectId); + expect(res.body).not.toHaveProperty("legacyEnvironmentId"); + expect(res.body).not.toHaveProperty("environmentId"); + }); }); describeEmbeddedPostgres("private project route visibility", { concurrent: false }, () => { diff --git a/server/src/services/projects.ts b/server/src/services/projects.ts index 48edc2ed5c5..616d245dd62 100644 --- a/server/src/services/projects.ts +++ b/server/src/services/projects.ts @@ -149,7 +149,7 @@ export async function ensurePersonalPrivateProject(dbOrTx: any, companyId: strin return project; } -interface ProjectWithGoals extends Omit { +interface ProjectWithGoals extends Omit { urlKey: string; goalIds: string[]; goals: ProjectGoalRef[]; @@ -201,8 +201,9 @@ async function attachGoals(db: Db, rows: ProjectRow[]): Promise { const g = map.get(r.id) ?? []; + const { legacyEnvironmentId: _legacyEnvironmentId, ...project } = r; return { - ...r, + ...project, urlKey: deriveProjectUrlKey(r.name, r.id), goalIds: g.map((x) => x.id), goals: g, @@ -1115,7 +1116,9 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyStatusCardIssueChange(tx, project.companyId); const [row] = await tx.delete(projects).where(eq(projects.id, id)).returning(); - return row ? { ...row, urlKey: deriveProjectUrlKey(row.name, row.id) } : null; + if (!row) return null; + const { legacyEnvironmentId: _legacyEnvironmentId, ...publicProject } = row; + return { ...publicProject, urlKey: deriveProjectUrlKey(row.name, row.id) }; }), listWorkspaces: async (projectId: string): Promise => {