From b4f2e905c3ba0b71804c08bc006664bcfc6030a6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 6 Nov 2026 16:58:04 -0600 Subject: [PATCH] Prepare online ephemeral enrollment through signed Cloud control Co-Authored-By: Paperclip --- ...-existing-workspace-ephemeral-migration.md | 26 ++++++ server/src/app.ts | 2 + server/src/middleware/cloud-control.ts | 1 + server/src/middleware/http-log-policy.ts | 2 + .../src/routes/ephemeral-enrollment.test.ts | 81 +++++++++++++++++++ server/src/routes/ephemeral-enrollment.ts | 17 ++++ server/src/services/cloud-runtime-identity.ts | 1 + .../src/services/ephemeral-enrollment.test.ts | 72 +++++++++++++++++ server/src/services/ephemeral-enrollment.ts | 33 ++++++++ 9 files changed, 235 insertions(+) create mode 100644 server/src/routes/ephemeral-enrollment.test.ts create mode 100644 server/src/routes/ephemeral-enrollment.ts create mode 100644 server/src/services/ephemeral-enrollment.test.ts create mode 100644 server/src/services/ephemeral-enrollment.ts diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 29c0d19f613..847a74433bb 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -5,6 +5,32 @@ rolling deployment while an agent works and new work is admitted. Enrollment mus preserve existing task files and historical recovery sources. It must not reset task workspaces, manufacture successful copyback, or weaken migration guards. +## Online enrollment preparation + +`POST /api/instance/ephemeral-enrollment/prepare` accepts only a fresh, stack-bound +Cloud control assertion naming `ephemeral-enrollment:prepare`. Tenant sessions, +including instance admins, cannot export the original decision signing key. +The strict request contains only an `attemptId` UUID. The owner first checks +durable-data eligibility, validates the replacement's ephemeral configuration, +and audits the attempt without storing its secret. It never creates a missing +signing key or changes admission, process configuration, or workspace bindings. + +The private, non-cacheable response binds the original key and its digest to the +attempt, stack, stamped Core commit, Railway deployment, observation time, and +current static ephemeral-mode setting. Cloud must deliver the key directly to +its existing secret vault; do not print, persist in registry metadata, or return +this response through an admin/browser API. HTTP diagnostics treat the entire +enrollment namespace as secret-sensitive. + +This is an advisory preparation receipt, always `enrollmentReady: false`. It is +not a writer fence, ownership-transfer receipt, or permission to detach a volume. +The remaining Cloud transition must arm static ephemeral mode while retaining +the original volume, verify that deployment and cessation of earlier local +writers, repeat durable eligibility checks on the armed deployment, and use the +qualified native handoff before detachment. Retain the original volume. Ordinary +release selection must remain unpinned. No step may substitute a global-idle +requirement for these ownership and durability checks. + ## Evidence and remaining work | Required outcome | Current evidence | Remaining gap | Owner / next action | diff --git a/server/src/app.ts b/server/src/app.ts index d6c677506cd..394e5e05504 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -155,6 +155,7 @@ import { primaryAgentRoutes } from "./routes/primary-agent.js"; import { resourceMembershipRoutes } from "./routes/resource-memberships.js"; import { inboxDismissalRoutes } from "./routes/inbox-dismissals.js"; import { instanceSettingsRoutes } from "./routes/instance-settings.js"; +import { ephemeralEnrollmentRoutes } from "./routes/ephemeral-enrollment.js"; import { instanceSettingsService } from "./services/instance-settings.js"; import { openApiRoutes } from "./routes/openapi.js"; import { @@ -841,6 +842,7 @@ export async function createApp( api.use(workspacePreferenceRoutes(db)); api.use(inboxDismissalRoutes(db)); api.use(instanceSettingsRoutes(db, workerManager, opts.prepareIdleDatabaseBackup, opts.idleDeadlineOwners)); + api.use(ephemeralEnrollmentRoutes(db)); if (opts.databaseBackupService) { api.use(instanceDatabaseBackupRoutes(opts.databaseBackupService)); } diff --git a/server/src/middleware/cloud-control.ts b/server/src/middleware/cloud-control.ts index 6165af6b0b6..276ab5cfcd2 100644 --- a/server/src/middleware/cloud-control.ts +++ b/server/src/middleware/cloud-control.ts @@ -16,6 +16,7 @@ export function verifiedIdleCapabilities(req: Request): VerifiedIdleCapabilities /** Endpoint → method → the one action a control assertion must name to take it. */ const ACTIONS_BY_ENDPOINT: Record> = { + "/api/instance/ephemeral-enrollment/prepare": { POST: "ephemeral-enrollment:prepare" }, "/api/instance/connector-events/wake": { POST: "connector-events:wake" }, "/api/instance/task-drain": { GET: "task-drain:read", diff --git a/server/src/middleware/http-log-policy.ts b/server/src/middleware/http-log-policy.ts index ef98af0e0ab..b9d2098ee7f 100644 --- a/server/src/middleware/http-log-policy.ts +++ b/server/src/middleware/http-log-policy.ts @@ -43,6 +43,7 @@ function normalizePath(url: string): string { * Labels come from this closed list, never from the request URL. */ export type SecretSensitiveRouteFamily = + | "ephemeral_enrollment" | "provider_webhook" | "muse" | "mcp_files" @@ -133,6 +134,7 @@ export function secretSensitiveRouteFamily( method: string | undefined, url: string | undefined, ): SecretSensitiveRouteFamily | null { + if (url && /^\/api\/instance\/ephemeral-enrollment(?:\/|$)/i.test(normalizePath(url).replace(/^https?:\/\/[^/]*/i, ""))) return "ephemeral_enrollment"; if (isPrivateWebhookHttpRequest(method, url)) return "provider_webhook"; if (isPrivateMuseHttpRequest(url)) return "muse"; if (url && normalizePath(url).startsWith("/mcp/files/")) return "mcp_files"; diff --git a/server/src/routes/ephemeral-enrollment.test.ts b/server/src/routes/ephemeral-enrollment.test.ts new file mode 100644 index 00000000000..83c860743ad --- /dev/null +++ b/server/src/routes/ephemeral-enrollment.test.ts @@ -0,0 +1,81 @@ +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { ephemeralEnrollmentRoutes } from "./ephemeral-enrollment.js"; +import { cloudControlMiddleware } from "../middleware/cloud-control.js"; +import { isSecretSensitiveHttpRequest } from "../middleware/http-log-policy.js"; +import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, CLOUD_CONTROL_JWS_TYPE, + CLOUD_RUNTIME_IDENTITY_ISSUER, resetCloudControlReplayFenceForTests } from "../services/cloud-runtime-identity.js"; + +const prepare = vi.hoisted(() => vi.fn()); +vi.mock("../services/ephemeral-enrollment.js", () => ({ prepareOnlineEphemeralEnrollment: prepare })); +const pair = generateKeyPairSync("ed25519"); +const jwk = { ...pair.publicKey.export({ format: "jwk" }), kid: "enrollment-fixture", alg: "EdDSA", use: "sig" }; +const path = "/api/instance/ephemeral-enrollment/prepare"; +const attemptId = randomUUID(); +const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url"); +function assertion(action = "ephemeral-enrollment:prepare", stackId = "stack-fixture") { + const iat = Math.floor(Date.now() / 1000); + const body = `${encode({ alg: "EdDSA", typ: CLOUD_CONTROL_JWS_TYPE, kid: jwk.kid })}.${encode({ + v: 1, iss: CLOUD_RUNTIME_IDENTITY_ISSUER, aud: CLOUD_CONTROL_AUDIENCE, + sub: stackId, action, requestId: randomUUID(), iat, exp: iat + 60, + })}`; + return `${body}.${sign(null, Buffer.from(body, "ascii"), pair.privateKey).toString("base64url")}`; +} +function app(tenant = false) { + const result = express(); result.use(express.json()); + result.use((req, _res, next) => { + req.actor = tenant ? { type: "board", source: "cloud_tenant", userId: "fixture-user", isInstanceAdmin: true } + : { type: "none", source: "none" }; + next(); + }); + result.use(cloudControlMiddleware()); result.use("/api", ephemeralEnrollmentRoutes({} as Db)); + result.use((error: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => { + res.status(400).json({ error: "fixture_validation_failed" }); + }); + return result; +} +beforeEach(() => { + vi.resetAllMocks(); resetCloudControlReplayFenceForTests(); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "stack-fixture"); + vi.stubEnv("PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS", JSON.stringify({ keys: [jwk] })); + prepare.mockResolvedValue({ attemptId, enrollmentReady: false, decisionSigningSecret: "fixture-original-key" }); +}); +afterEach(() => vi.unstubAllEnvs()); + +it("exports once only to a fresh stack-bound signed prepare assertion", async () => { + const server = app(), token = assertion(); + const accepted = await request(server).post(path).set(CLOUD_CONTROL_HEADER, token).send({ attemptId }); + expect(accepted.status).toBe(200); + expect(accepted.headers["cache-control"]).toBe("no-store"); + expect(accepted.body).toMatchObject({ decisionSigningSecret: "fixture-original-key", enrollmentReady: false }); + expect((await request(server).post(path).set(CLOUD_CONTROL_HEADER, token).send({ attemptId })).status).toBe(401); + expect(prepare).toHaveBeenCalledOnce(); +}); + +it.each([false, true])("refuses an unsigned caller, including a tenant instance admin (%s)", async tenant => { + expect((await request(app(tenant)).post(path).send({ attemptId })).status).toBe(403); + expect(prepare).not.toHaveBeenCalled(); +}); + +it.each([["task-drain:read", "stack-fixture"], ["ephemeral-enrollment:prepare", "stack-other"]])( + "refuses mismatched action or stack (%s, %s)", async (action, stack) => { + const result = await request(app()).post(path).set(CLOUD_CONTROL_HEADER, assertion(action, stack)).send({ attemptId }); + expect(result.status).toBe(401); expect(prepare).not.toHaveBeenCalled(); + }); + +it("rejects malformed or additional caller-supplied authority before export", async () => { + for (const body of [{ attemptId: "bad" }, { attemptId, stackId: "stack-other" }]) { + expect((await request(app()).post(path).set(CLOUD_CONTROL_HEADER, assertion()).send(body)).status).toBe(400); + } + expect(prepare).not.toHaveBeenCalled(); +}); + +it("keeps enrollment HTTP diagnostics secret-sensitive, including rejected methods", () => { + for (const method of ["POST", "GET", "DELETE"]) { + expect(isSecretSensitiveHttpRequest(method, path)).toBe(true); + expect(isSecretSensitiveHttpRequest(method, `https://fixture.test${path}`)).toBe(true); + } +}); diff --git a/server/src/routes/ephemeral-enrollment.ts b/server/src/routes/ephemeral-enrollment.ts new file mode 100644 index 00000000000..82d1e52a7b5 --- /dev/null +++ b/server/src/routes/ephemeral-enrollment.ts @@ -0,0 +1,17 @@ +import { Router } from "express"; +import { z } from "zod"; +import type { Db } from "@paperclipai/db"; +import { validate } from "../middleware/validate.js"; +import { prepareOnlineEphemeralEnrollment } from "../services/ephemeral-enrollment.js"; + +export function ephemeralEnrollmentRoutes(db: Db) { + const router = Router(); + router.post("/instance/ephemeral-enrollment/prepare", (req, res, next) => { + res.set("Cache-Control", "no-store"); + if (req.actor.source !== "cloud_control") { res.status(403).json({ error: "cloud_control_required" }); return; } + next(); + }, validate(z.object({ attemptId: z.string().uuid() }).strict()), async (req, res) => { + res.json(await prepareOnlineEphemeralEnrollment(db, req.body.attemptId)); + }); + return router; +} diff --git a/server/src/services/cloud-runtime-identity.ts b/server/src/services/cloud-runtime-identity.ts index cffe6a8dbfd..9738550a2da 100644 --- a/server/src/services/cloud-runtime-identity.ts +++ b/server/src/services/cloud-runtime-identity.ts @@ -447,6 +447,7 @@ export const CLOUD_CONTROL_HEADER = "x-paperclip-cloud-control"; export const CLOUD_CONTROL_AUDIENCE = "paperclip-cloud-control/v1"; export const CLOUD_CONTROL_JWS_TYPE = "paperclip-cloud-control+jwt"; export const CLOUD_CONTROL_ACTIONS = [ + "ephemeral-enrollment:prepare", "task-drain:read", "task-drain:start", "task-drain:stop", diff --git a/server/src/services/ephemeral-enrollment.test.ts b/server/src/services/ephemeral-enrollment.test.ts new file mode 100644 index 00000000000..507defddde5 --- /dev/null +++ b/server/src/services/ephemeral-enrollment.test.ts @@ -0,0 +1,72 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { prepareOnlineEphemeralEnrollment } from "./ephemeral-enrollment.js"; + +const owners = vi.hoisted(() => ({ eligibility: vi.fn(), key: vi.fn(), audit: vi.fn() })); +vi.mock("./ephemeral-server-contract.js", async original => ({ + ...await original(), readEphemeralServerDataEligibility: owners.eligibility, +})); +vi.mock("./decision-signing.js", () => ({ resolveDecisionSigningSecret: owners.key })); +vi.mock("./activity-log.js", () => ({ logActivity: owners.audit })); +vi.mock("../config.js", () => ({ loadConfig: () => ({ databaseUrl: "postgres://fixture:fixture@db.test/database", storageProvider: "s3" }) })); + +const db = {} as Db; +const attempt = "12345678-1234-4234-8234-123456789abc"; +const key = "fixture-decision-signing-key-0123456789"; +beforeEach(() => { + vi.resetAllMocks(); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-managed-instance"); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "stack-fixture"); + vi.stubEnv("PAPERCLIP_BUILD_COMMIT", "a".repeat(40)); + vi.stubEnv("RAILWAY_DEPLOYMENT_ID", "11111111-1111-4111-8111-111111111111"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.stubEnv("PAPERCLIP_SHARED_UI_ASSETS", "true"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + vi.stubEnv("PAPERCLIP_DATABASE_LISTEN_URL", "postgres://fixture:fixture@db.test/database"); + owners.eligibility.mockResolvedValue({ eligible: true, blockers: [] }); + owners.key.mockReturnValue(key); + owners.audit.mockResolvedValue(undefined); +}); +afterEach(() => vi.unstubAllEnvs()); + +it("exports the original key only after owner eligibility, config validation and a secret-free audit", async () => { + const result = await prepareOnlineEphemeralEnrollment(db, attempt); + expect(result).toMatchObject({ attemptId: attempt, stackId: "stack-fixture", commit: "a".repeat(40), + deploymentId: "11111111-1111-4111-8111-111111111111", ephemeralServer: false, enrollmentReady: false, + decisionSigningSecret: key }); + expect(result.decisionKeyDigest).toMatch(/^[a-f0-9]{64}$/); + expect(owners.key).toHaveBeenCalledWith({ allowCreate: false }); + expect(owners.audit).toHaveBeenCalledOnce(); + expect(JSON.stringify(owners.audit.mock.calls)).not.toContain(key); + expect(process.env.PAPERCLIP_EPHEMERAL_SERVER).toBe("false"); +}); + +it("does not read or export the key while a local obligation remains", async () => { + owners.eligibility.mockResolvedValue({ eligible: false, blockers: ["workspaces_require_migration"] }); + await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toMatchObject({ status: 409, + details: { blockers: ["workspaces_require_migration"] } }); + expect(owners.key).not.toHaveBeenCalled(); expect(owners.audit).not.toHaveBeenCalled(); +}); + +it("refuses an invalid replacement configuration without publishing or auditing a key", async () => { + vi.stubEnv("PAPERCLIP_DATABASE_LISTEN_URL", "postgres://fixture:fixture@other.test/database"); + await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toThrow("direct connection to the same database"); + expect(owners.audit).not.toHaveBeenCalled(); +}); + +it("fails closed if key preservation auditing fails", async () => { + owners.audit.mockRejectedValue(new Error("fixture_audit_failed")); + await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toThrow("fixture_audit_failed"); +}); + +it("identifies the armed deployment without granting detachment authority", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + expect(await prepareOnlineEphemeralEnrollment(db, attempt)).toMatchObject({ ephemeralServer: true, enrollmentReady: false }); +}); + +it("refuses an unknown deployment before inspecting data or keys", async () => { + vi.stubEnv("RAILWAY_DEPLOYMENT_ID", ""); + await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toMatchObject({ status: 409 }); + expect(owners.eligibility).not.toHaveBeenCalled(); expect(owners.key).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/ephemeral-enrollment.ts b/server/src/services/ephemeral-enrollment.ts new file mode 100644 index 00000000000..95e0a210be2 --- /dev/null +++ b/server/src/services/ephemeral-enrollment.ts @@ -0,0 +1,33 @@ +import { createHash } from "node:crypto"; +import type { Db } from "@paperclipai/db"; +import { readBuildCommit } from "../build-commit.js"; +import { loadConfig } from "../config.js"; +import { conflict } from "../errors.js"; +import { logActivity } from "./activity-log.js"; +import { cloudTenantPrimaryCompanyId, getCloudStackContext } from "./cloud-instance.js"; +import { resolveDecisionSigningSecret } from "./decision-signing.js"; +import { assertEphemeralServerConfig, ephemeralServerEnabled, readEphemeralServerDataEligibility } from "./ephemeral-server-contract.js"; + +export async function prepareOnlineEphemeralEnrollment(db: Db, attemptId: string) { + const stackId = getCloudStackContext()?.stackId; + const commit = readBuildCommit(); + const deploymentId = process.env.RAILWAY_DEPLOYMENT_ID; + if (!stackId || !commit || !deploymentId || !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(deploymentId)) { + throw conflict("Online enrollment requires a stamped managed Railway deployment"); + } + const eligibility = await readEphemeralServerDataEligibility(db); + if (!eligibility.eligible) throw conflict("Durable data migration is incomplete", { blockers: eligibility.blockers }); + const decisionSigningSecret = resolveDecisionSigningSecret({ allowCreate: false }); + assertEphemeralServerConfig(loadConfig(), { ...process.env, PAPERCLIP_EPHEMERAL_SERVER: "true", + PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret }); + const receipt = { + schema: "paperclip.online-ephemeral-enrollment.v1" as const, + attemptId, stackId, commit, deploymentId, observedAt: new Date().toISOString(), + ephemeralServer: ephemeralServerEnabled(), + decisionKeyDigest: createHash("sha256").update(decisionSigningSecret).digest("hex"), + enrollmentReady: false as const, + }; + await logActivity(db, { companyId: cloudTenantPrimaryCompanyId(stackId), actorType: "system", actorId: "paperclip-cloud", + action: "instance.ephemeral_enrollment.prepared", entityType: "instance", entityId: stackId, details: receipt }); + return { ...receipt, decisionSigningSecret }; +}