diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 847a74433bb..898ca21cfde 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -14,6 +14,16 @@ The strict request contains only an `attemptId` UUID. The owner first checks durable-data eligibility, validates the replacement's ephemeral configuration, and audits the attempt without storing its secret. It never creates a missing signing key or changes admission, process configuration, or workspace bindings. +Preparation uses startup durability checks, so ordinary admission before skill +pins are saved does not impose a global-idle requirement. Retention and migration +checks keep their stronger pin requirements. Actual admitted local skill assets +still require their owning preservation proof before volume removal. + +The existing audit owner accepts a caller-bound receipt UUID. Preparation derives +it from the verified signed request ID and stack under its own namespace. The +database primary key atomically consumes that assertion before returning the key, +including concurrent replicas and process restarts. The process-local assertion +fence remains an early refusal. Expired assertions cannot return a secret. The private, non-cacheable response binds the original key and its digest to the attempt, stack, stamped Core commit, Railway deployment, observation time, and diff --git a/server/src/__tests__/ephemeral-enrollment-replay.test.ts b/server/src/__tests__/ephemeral-enrollment-replay.test.ts new file mode 100644 index 00000000000..351d5720613 --- /dev/null +++ b/server/src/__tests__/ephemeral-enrollment-replay.test.ts @@ -0,0 +1,69 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { activityLog, agents, companies, companySkills, companySkillVersions, createDb, heartbeatRuns, type Db } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { prepareOnlineEphemeralEnrollment } from "../services/ephemeral-enrollment.js"; +import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js"; +import { initializeDurableAgentFiles } from "../services/agent-file-durability.js"; +import { resetCloudControlReplayFenceForTests } from "../services/cloud-runtime-identity.js"; + +const config = vi.hoisted(() => ({ databaseUrl: "", storageProvider: "s3" })); +vi.mock("../config.js", () => ({ loadConfig: () => config })); +vi.mock("../storage/index.js", () => ({ getStorageService: () => ({ provider: "s3" }) })); +let database: Awaited>; +let db: ReturnType; +beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("ephemeral-enrollment-replay-"); + db = createDb(database.connectionString); config.databaseUrl = database.connectionString; +}, 90_000); +afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); +afterEach(() => vi.unstubAllEnvs()); + +async function fixture() { + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + const stackId = `stack-fixture-${randomUUID()}`, companyId = cloudTenantPrimaryCompanyId(stackId), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Online enrollment fixture", issuePrefix: `P${randomUUID().slice(0, 7).toUpperCase()}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Preparing agent" }); + await db.transaction(tx => initializeDurableAgentFiles(tx as unknown as Db, { id: agentId, companyId })); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-managed-instance"); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", stackId); + vi.stubEnv("PAPERCLIP_BUILD_COMMIT", "a".repeat(40)); + vi.stubEnv("RAILWAY_DEPLOYMENT_ID", randomUUID()); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.stubEnv("PAPERCLIP_SHARED_UI_ASSETS", "true"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + vi.stubEnv("PAPERCLIP_DECISION_SIGNING_SECRET", "fixture-preserved-original-key-at-least-32"); + vi.stubEnv("PAPERCLIP_DATABASE_LISTEN_URL", database.connectionString); + return { companyId, agentId, attemptId: randomUUID(), control: { requestId: randomUUID(), exp: Math.floor(Date.now() / 1000) + 60 } }; +} + +it("exports once across simultaneous callers and a cleared process replay map using the durable audit key", async () => { + const f = await fixture(); + const results = await Promise.allSettled([prepareOnlineEphemeralEnrollment(db, f.attemptId, f.control), + prepareOnlineEphemeralEnrollment(db, f.attemptId, f.control)]); + if (!results.some(result => result.status === "fulfilled")) throw new AggregateError(results.map(result => (result as PromiseRejectedResult).reason), "All preparation callers failed"); + expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1); + const rejected = results.find(result => result.status === "rejected") as PromiseRejectedResult; + expect(rejected.reason).toMatchObject({ status: 409 }); + resetCloudControlReplayFenceForTests(); + await expect(prepareOnlineEphemeralEnrollment(db, f.attemptId, f.control)).rejects.toMatchObject({ status: 409 }); + const audit = await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId)); + expect(audit).toHaveLength(1); + expect(JSON.stringify(audit)).not.toContain(process.env.PAPERCLIP_DECISION_SIGNING_SECRET); +}); + +it("prepares during normal admission before skill pins are saved, while preserving installation durability", async () => { + const f = await fixture(), skillId = randomUUID(), versionId = randomUUID(); + await db.insert(companySkills).values({ id: skillId, companyId: f.companyId, key: "overlap", slug: "overlap", name: "Overlap", + sourceType: "managed", markdown: "# Overlap", fileInventory: [{ path: "SKILL.md", kind: "entrypoint" }] }); + await db.insert(companySkillVersions).values({ id: versionId, companyId: f.companyId, companySkillId: skillId, revisionNumber: 1, + fileInventory: [{ path: "SKILL.md", kind: "skill", content: "# Overlap" }] }); + await db.update(companySkills).set({ currentVersionId: versionId }).where(eq(companySkills.id, skillId)); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, invocationSource: "on_demand", status: "running", contextSnapshot: {} }); + expect(await prepareOnlineEphemeralEnrollment(db, f.attemptId, f.control)).toMatchObject({ enrollmentReady: false }); + await db.update(companySkills).set({ currentVersionId: null }).where(eq(companySkills.id, skillId)); + await expect(prepareOnlineEphemeralEnrollment(db, f.attemptId, { ...f.control, requestId: randomUUID() })).rejects.toMatchObject({ status: 409, + details: { blockers: ["skills_require_migration"] } }); +}); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 93d9f631308..4969f71ca07 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -49,6 +49,7 @@ const apiPrefixes: Record = { "company-skill-policy.ts": "/api", "connection-intents.ts": "/api", "connection-event-wake.ts": "/", + "ephemeral-enrollment.ts": "/api", "costs.ts": "/api", "dashboard.ts": "/api", "dot-runner.ts": "/api", @@ -118,6 +119,7 @@ const explicitOpenApiCoverageExclusions = new Set(); const explicitOpenApiOperationCoverageExclusions = new Set([ // Exact Cloud-control signed action; ordinary board/agent credentials cannot invoke it. "POST /api/instance/connector-events/wake", + "POST /api/instance/ephemeral-enrollment/prepare", // Muse uses its closed, versioned receiver protocol and dedicated credentials, // documented in doc/muse-personal-agent.md and the shared protocol contract. // Board connection/qualification management remains in this OpenAPI document. diff --git a/server/src/middleware/cloud-control.ts b/server/src/middleware/cloud-control.ts index 276ab5cfcd2..90a6ba00936 100644 --- a/server/src/middleware/cloud-control.ts +++ b/server/src/middleware/cloud-control.ts @@ -6,9 +6,14 @@ import { CLOUD_CONTROL_HEADER, verifyCloudControlAssertion, type CloudControlAction, + type CloudControlClaims, } from "../services/cloud-runtime-identity.js"; const idleCapabilities = new WeakMap(); +const controlClaims = new WeakMap(); +export function verifiedCloudControlClaims(req: Request): CloudControlClaims | undefined { + return controlClaims.get(req); +} /** Only this middleware's verified assertion can install request capabilities. */ export function verifiedIdleCapabilities(req: Request): VerifiedIdleCapabilities | undefined { return idleCapabilities.get(req); @@ -60,6 +65,7 @@ export function cloudControlMiddleware(): RequestHandler { } try { const claims = verifyCloudControlAssertion({ compactJws: assertion, expectedAction }); + controlClaims.set(req, claims); if (claims.idleCapabilities) idleCapabilities.set(req, claims.idleCapabilities as VerifiedIdleCapabilities); } catch (error) { logger.warn({ err: error }, "Rejected Cloud control assertion"); diff --git a/server/src/routes/ephemeral-enrollment.ts b/server/src/routes/ephemeral-enrollment.ts index 82d1e52a7b5..b626232c9f7 100644 --- a/server/src/routes/ephemeral-enrollment.ts +++ b/server/src/routes/ephemeral-enrollment.ts @@ -2,6 +2,7 @@ import { Router } from "express"; import { z } from "zod"; import type { Db } from "@paperclipai/db"; import { validate } from "../middleware/validate.js"; +import { verifiedCloudControlClaims } from "../middleware/cloud-control.js"; import { prepareOnlineEphemeralEnrollment } from "../services/ephemeral-enrollment.js"; export function ephemeralEnrollmentRoutes(db: Db) { @@ -11,7 +12,9 @@ export function ephemeralEnrollmentRoutes(db: Db) { if (req.actor.source !== "cloud_control") { res.status(403).json({ error: "cloud_control_required" }); return; } next(); }, validate(z.object({ attemptId: z.string().uuid() }).strict()), async (req, res) => { - res.json(await prepareOnlineEphemeralEnrollment(db, req.body.attemptId)); + const claims = verifiedCloudControlClaims(req); + if (claims?.action !== "ephemeral-enrollment:prepare") { res.status(403).json({ error: "cloud_control_required" }); return; } + res.json(await prepareOnlineEphemeralEnrollment(db, req.body.attemptId, claims)); }); return router; } diff --git a/server/src/services/activity-log.ts b/server/src/services/activity-log.ts index 4e3aab320b0..de1940acd24 100644 --- a/server/src/services/activity-log.ts +++ b/server/src/services/activity-log.ts @@ -164,10 +164,11 @@ export function publishActivity(publication: ActivityPublication) { } finally { finish(); } } -export async function persistActivity(db: Db, input: LogActivityInput) { +export async function persistActivity(db: Db, input: LogActivityInput, receipt?: { id: string }) { const redactedDetails = await redactActivityDetails(db, input.authorization ? { ...input.details, permissionDecision: input.authorization, ...(input.agentApiKeyId ? { agentApiKeyId: input.agentApiKeyId } : {}) } : input.details ?? null); const responsibleUserId = await resolveResponsibleUserIdForActivity(db, input); const insert = (runner: Db) => runner.insert(activityLog).values({ + ...(receipt ? { id: receipt.id } : {}), companyId: input.companyId, actorType: input.actorType, actorId: input.actorId, diff --git a/server/src/services/ephemeral-enrollment.test.ts b/server/src/services/ephemeral-enrollment.test.ts index 507defddde5..f638e2109cf 100644 --- a/server/src/services/ephemeral-enrollment.test.ts +++ b/server/src/services/ephemeral-enrollment.test.ts @@ -7,14 +7,16 @@ vi.mock("./ephemeral-server-contract.js", async original => ({ ...await original(), readEphemeralServerDataEligibility: owners.eligibility, })); vi.mock("./decision-signing.js", () => ({ resolveDecisionSigningSecret: owners.key })); -vi.mock("./activity-log.js", () => ({ logActivity: owners.audit })); +vi.mock("./activity-log.js", () => ({ persistActivity: owners.audit, publishActivity: vi.fn() })); vi.mock("../config.js", () => ({ loadConfig: () => ({ databaseUrl: "postgres://fixture:fixture@db.test/database", storageProvider: "s3" }) })); const db = {} as Db; const attempt = "12345678-1234-4234-8234-123456789abc"; const key = "fixture-decision-signing-key-0123456789"; +const control = { requestId: "fixture-signed-request", exp: 0 }; beforeEach(() => { vi.resetAllMocks(); + control.exp = Math.floor(Date.now() / 1000) + 60; vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-managed-instance"); vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "stack-fixture"); vi.stubEnv("PAPERCLIP_BUILD_COMMIT", "a".repeat(40)); @@ -26,17 +28,18 @@ beforeEach(() => { vi.stubEnv("PAPERCLIP_DATABASE_LISTEN_URL", "postgres://fixture:fixture@db.test/database"); owners.eligibility.mockResolvedValue({ eligible: true, blockers: [] }); owners.key.mockReturnValue(key); - owners.audit.mockResolvedValue(undefined); + owners.audit.mockResolvedValue({ publication: {} }); }); afterEach(() => vi.unstubAllEnvs()); it("exports the original key only after owner eligibility, config validation and a secret-free audit", async () => { - const result = await prepareOnlineEphemeralEnrollment(db, attempt); + const result = await prepareOnlineEphemeralEnrollment(db, attempt, control); expect(result).toMatchObject({ attemptId: attempt, stackId: "stack-fixture", commit: "a".repeat(40), deploymentId: "11111111-1111-4111-8111-111111111111", ephemeralServer: false, enrollmentReady: false, decisionSigningSecret: key }); expect(result.decisionKeyDigest).toMatch(/^[a-f0-9]{64}$/); expect(owners.key).toHaveBeenCalledWith({ allowCreate: false }); + expect(owners.eligibility).toHaveBeenCalledWith(db, { purpose: "startup" }); expect(owners.audit).toHaveBeenCalledOnce(); expect(JSON.stringify(owners.audit.mock.calls)).not.toContain(key); expect(process.env.PAPERCLIP_EPHEMERAL_SERVER).toBe("false"); @@ -44,29 +47,35 @@ it("exports the original key only after owner eligibility, config validation and it("does not read or export the key while a local obligation remains", async () => { owners.eligibility.mockResolvedValue({ eligible: false, blockers: ["workspaces_require_migration"] }); - await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toMatchObject({ status: 409, + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toMatchObject({ status: 409, details: { blockers: ["workspaces_require_migration"] } }); expect(owners.key).not.toHaveBeenCalled(); expect(owners.audit).not.toHaveBeenCalled(); }); it("refuses an invalid replacement configuration without publishing or auditing a key", async () => { vi.stubEnv("PAPERCLIP_DATABASE_LISTEN_URL", "postgres://fixture:fixture@other.test/database"); - await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toThrow("direct connection to the same database"); + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toThrow("direct connection to the same database"); expect(owners.audit).not.toHaveBeenCalled(); }); it("fails closed if key preservation auditing fails", async () => { owners.audit.mockRejectedValue(new Error("fixture_audit_failed")); - await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toThrow("fixture_audit_failed"); + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toThrow("fixture_audit_failed"); }); it("identifies the armed deployment without granting detachment authority", async () => { vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); - expect(await prepareOnlineEphemeralEnrollment(db, attempt)).toMatchObject({ ephemeralServer: true, enrollmentReady: false }); + expect(await prepareOnlineEphemeralEnrollment(db, attempt, control)).toMatchObject({ ephemeralServer: true, enrollmentReady: false }); }); it("refuses an unknown deployment before inspecting data or keys", async () => { vi.stubEnv("RAILWAY_DEPLOYMENT_ID", ""); - await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toMatchObject({ status: 409 }); + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toMatchObject({ status: 409 }); expect(owners.eligibility).not.toHaveBeenCalled(); expect(owners.key).not.toHaveBeenCalled(); }); + +it("refuses an expired assertion after inspection without exporting", async () => { + control.exp = Math.floor(Date.now() / 1000) - 1; + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toMatchObject({ status: 401 }); + expect(owners.audit).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/ephemeral-enrollment.ts b/server/src/services/ephemeral-enrollment.ts index 95e0a210be2..415fdad5243 100644 --- a/server/src/services/ephemeral-enrollment.ts +++ b/server/src/services/ephemeral-enrollment.ts @@ -2,20 +2,21 @@ import { createHash } from "node:crypto"; import type { Db } from "@paperclipai/db"; import { readBuildCommit } from "../build-commit.js"; import { loadConfig } from "../config.js"; -import { conflict } from "../errors.js"; -import { logActivity } from "./activity-log.js"; +import { conflict, unauthorized } from "../errors.js"; +import { persistActivity, publishActivity } from "./activity-log.js"; +import type { CloudControlClaims } from "./cloud-runtime-identity.js"; import { cloudTenantPrimaryCompanyId, getCloudStackContext } from "./cloud-instance.js"; import { resolveDecisionSigningSecret } from "./decision-signing.js"; import { assertEphemeralServerConfig, ephemeralServerEnabled, readEphemeralServerDataEligibility } from "./ephemeral-server-contract.js"; -export async function prepareOnlineEphemeralEnrollment(db: Db, attemptId: string) { +export async function prepareOnlineEphemeralEnrollment(db: Db, attemptId: string, control: Pick) { const stackId = getCloudStackContext()?.stackId; const commit = readBuildCommit(); const deploymentId = process.env.RAILWAY_DEPLOYMENT_ID; if (!stackId || !commit || !deploymentId || !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(deploymentId)) { throw conflict("Online enrollment requires a stamped managed Railway deployment"); } - const eligibility = await readEphemeralServerDataEligibility(db); + const eligibility = await readEphemeralServerDataEligibility(db, { purpose: "startup" }); if (!eligibility.eligible) throw conflict("Durable data migration is incomplete", { blockers: eligibility.blockers }); const decisionSigningSecret = resolveDecisionSigningSecret({ allowCreate: false }); assertEphemeralServerConfig(loadConfig(), { ...process.env, PAPERCLIP_EPHEMERAL_SERVER: "true", @@ -27,7 +28,20 @@ export async function prepareOnlineEphemeralEnrollment(db: Db, attemptId: string decisionKeyDigest: createHash("sha256").update(decisionSigningSecret).digest("hex"), enrollmentReady: false as const, }; - await logActivity(db, { companyId: cloudTenantPrimaryCompanyId(stackId), actorType: "system", actorId: "paperclip-cloud", - action: "instance.ephemeral_enrollment.prepared", entityType: "instance", entityId: stackId, details: receipt }); + if (control.exp * 1000 <= Date.now()) throw unauthorized("Cloud preparation assertion expired"); + const bytes = createHash("sha256").update(`paperclip-ephemeral-preparation:v1:${stackId}:${control.requestId}`).digest().subarray(0, 16); + bytes[6] = (bytes[6]! & 0x0f) | 0x50; bytes[8] = (bytes[8]! & 0x3f) | 0x80; + const hex = bytes.toString("hex"); + const id = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; + try { + const { publication } = await persistActivity(db, { companyId: cloudTenantPrimaryCompanyId(stackId), actorType: "system", actorId: "paperclip-cloud", + action: "instance.ephemeral_enrollment.prepared", entityType: "instance", entityId: stackId, details: receipt }, { id }); + publishActivity(publication); + } catch (error) { + const pg = (error as { cause?: { code?: string; constraint_name?: string } }).cause ?? error as { code?: string; constraint_name?: string }; + if (pg.code === "23505" && pg.constraint_name === "activity_log_pkey") throw conflict("Cloud preparation assertion was already used"); + throw error; + } + if (control.exp * 1000 <= Date.now()) throw unauthorized("Cloud preparation assertion expired"); return { ...receipt, decisionSigningSecret }; }