From 53467218476f42a780d2afbecf0c28861497a96d Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 6 Nov 2026 17:18:32 -0600 Subject: [PATCH] Consume preparation assertions in durable audit receipts Use startup durability eligibility for online preparation. Preserve exact request consumption across replicas and restarts through the existing audit primary key. Cover real database concurrency, admission-time pins and private-route coverage. Co-Authored-By: Paperclip --- ...-existing-workspace-ephemeral-migration.md | 10 +++ .../ephemeral-enrollment-replay.test.ts | 69 +++++++++++++++++++ server/src/__tests__/openapi-routes.test.ts | 2 + server/src/middleware/cloud-control.ts | 6 ++ server/src/routes/ephemeral-enrollment.ts | 5 +- server/src/services/activity-log.ts | 3 +- .../src/services/ephemeral-enrollment.test.ts | 25 ++++--- server/src/services/ephemeral-enrollment.ts | 26 +++++-- 8 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 server/src/__tests__/ephemeral-enrollment-replay.test.ts diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index 847a74433bb..898ca21cfde 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -14,6 +14,16 @@ The strict request contains only an `attemptId` UUID. The owner first checks durable-data eligibility, validates the replacement's ephemeral configuration, and audits the attempt without storing its secret. It never creates a missing signing key or changes admission, process configuration, or workspace bindings. +Preparation uses startup durability checks, so ordinary admission before skill +pins are saved does not impose a global-idle requirement. Retention and migration +checks keep their stronger pin requirements. Actual admitted local skill assets +still require their owning preservation proof before volume removal. + +The existing audit owner accepts a caller-bound receipt UUID. Preparation derives +it from the verified signed request ID and stack under its own namespace. The +database primary key atomically consumes that assertion before returning the key, +including concurrent replicas and process restarts. The process-local assertion +fence remains an early refusal. Expired assertions cannot return a secret. The private, non-cacheable response binds the original key and its digest to the attempt, stack, stamped Core commit, Railway deployment, observation time, and diff --git a/server/src/__tests__/ephemeral-enrollment-replay.test.ts b/server/src/__tests__/ephemeral-enrollment-replay.test.ts new file mode 100644 index 00000000000..351d5720613 --- /dev/null +++ b/server/src/__tests__/ephemeral-enrollment-replay.test.ts @@ -0,0 +1,69 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { activityLog, agents, companies, companySkills, companySkillVersions, createDb, heartbeatRuns, type Db } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { prepareOnlineEphemeralEnrollment } from "../services/ephemeral-enrollment.js"; +import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js"; +import { initializeDurableAgentFiles } from "../services/agent-file-durability.js"; +import { resetCloudControlReplayFenceForTests } from "../services/cloud-runtime-identity.js"; + +const config = vi.hoisted(() => ({ databaseUrl: "", storageProvider: "s3" })); +vi.mock("../config.js", () => ({ loadConfig: () => config })); +vi.mock("../storage/index.js", () => ({ getStorageService: () => ({ provider: "s3" }) })); +let database: Awaited>; +let db: ReturnType; +beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("ephemeral-enrollment-replay-"); + db = createDb(database.connectionString); config.databaseUrl = database.connectionString; +}, 90_000); +afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); +afterEach(() => vi.unstubAllEnvs()); + +async function fixture() { + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + const stackId = `stack-fixture-${randomUUID()}`, companyId = cloudTenantPrimaryCompanyId(stackId), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Online enrollment fixture", issuePrefix: `P${randomUUID().slice(0, 7).toUpperCase()}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Preparing agent" }); + await db.transaction(tx => initializeDurableAgentFiles(tx as unknown as Db, { id: agentId, companyId })); + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-managed-instance"); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", stackId); + vi.stubEnv("PAPERCLIP_BUILD_COMMIT", "a".repeat(40)); + vi.stubEnv("RAILWAY_DEPLOYMENT_ID", randomUUID()); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.stubEnv("PAPERCLIP_SHARED_UI_ASSETS", "true"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + vi.stubEnv("PAPERCLIP_DECISION_SIGNING_SECRET", "fixture-preserved-original-key-at-least-32"); + vi.stubEnv("PAPERCLIP_DATABASE_LISTEN_URL", database.connectionString); + return { companyId, agentId, attemptId: randomUUID(), control: { requestId: randomUUID(), exp: Math.floor(Date.now() / 1000) + 60 } }; +} + +it("exports once across simultaneous callers and a cleared process replay map using the durable audit key", async () => { + const f = await fixture(); + const results = await Promise.allSettled([prepareOnlineEphemeralEnrollment(db, f.attemptId, f.control), + prepareOnlineEphemeralEnrollment(db, f.attemptId, f.control)]); + if (!results.some(result => result.status === "fulfilled")) throw new AggregateError(results.map(result => (result as PromiseRejectedResult).reason), "All preparation callers failed"); + expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1); + const rejected = results.find(result => result.status === "rejected") as PromiseRejectedResult; + expect(rejected.reason).toMatchObject({ status: 409 }); + resetCloudControlReplayFenceForTests(); + await expect(prepareOnlineEphemeralEnrollment(db, f.attemptId, f.control)).rejects.toMatchObject({ status: 409 }); + const audit = await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId)); + expect(audit).toHaveLength(1); + expect(JSON.stringify(audit)).not.toContain(process.env.PAPERCLIP_DECISION_SIGNING_SECRET); +}); + +it("prepares during normal admission before skill pins are saved, while preserving installation durability", async () => { + const f = await fixture(), skillId = randomUUID(), versionId = randomUUID(); + await db.insert(companySkills).values({ id: skillId, companyId: f.companyId, key: "overlap", slug: "overlap", name: "Overlap", + sourceType: "managed", markdown: "# Overlap", fileInventory: [{ path: "SKILL.md", kind: "entrypoint" }] }); + await db.insert(companySkillVersions).values({ id: versionId, companyId: f.companyId, companySkillId: skillId, revisionNumber: 1, + fileInventory: [{ path: "SKILL.md", kind: "skill", content: "# Overlap" }] }); + await db.update(companySkills).set({ currentVersionId: versionId }).where(eq(companySkills.id, skillId)); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, invocationSource: "on_demand", status: "running", contextSnapshot: {} }); + expect(await prepareOnlineEphemeralEnrollment(db, f.attemptId, f.control)).toMatchObject({ enrollmentReady: false }); + await db.update(companySkills).set({ currentVersionId: null }).where(eq(companySkills.id, skillId)); + await expect(prepareOnlineEphemeralEnrollment(db, f.attemptId, { ...f.control, requestId: randomUUID() })).rejects.toMatchObject({ status: 409, + details: { blockers: ["skills_require_migration"] } }); +}); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 93d9f631308..4969f71ca07 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -49,6 +49,7 @@ const apiPrefixes: Record = { "company-skill-policy.ts": "/api", "connection-intents.ts": "/api", "connection-event-wake.ts": "/", + "ephemeral-enrollment.ts": "/api", "costs.ts": "/api", "dashboard.ts": "/api", "dot-runner.ts": "/api", @@ -118,6 +119,7 @@ const explicitOpenApiCoverageExclusions = new Set(); const explicitOpenApiOperationCoverageExclusions = new Set([ // Exact Cloud-control signed action; ordinary board/agent credentials cannot invoke it. "POST /api/instance/connector-events/wake", + "POST /api/instance/ephemeral-enrollment/prepare", // Muse uses its closed, versioned receiver protocol and dedicated credentials, // documented in doc/muse-personal-agent.md and the shared protocol contract. // Board connection/qualification management remains in this OpenAPI document. diff --git a/server/src/middleware/cloud-control.ts b/server/src/middleware/cloud-control.ts index 276ab5cfcd2..90a6ba00936 100644 --- a/server/src/middleware/cloud-control.ts +++ b/server/src/middleware/cloud-control.ts @@ -6,9 +6,14 @@ import { CLOUD_CONTROL_HEADER, verifyCloudControlAssertion, type CloudControlAction, + type CloudControlClaims, } from "../services/cloud-runtime-identity.js"; const idleCapabilities = new WeakMap(); +const controlClaims = new WeakMap(); +export function verifiedCloudControlClaims(req: Request): CloudControlClaims | undefined { + return controlClaims.get(req); +} /** Only this middleware's verified assertion can install request capabilities. */ export function verifiedIdleCapabilities(req: Request): VerifiedIdleCapabilities | undefined { return idleCapabilities.get(req); @@ -60,6 +65,7 @@ export function cloudControlMiddleware(): RequestHandler { } try { const claims = verifyCloudControlAssertion({ compactJws: assertion, expectedAction }); + controlClaims.set(req, claims); if (claims.idleCapabilities) idleCapabilities.set(req, claims.idleCapabilities as VerifiedIdleCapabilities); } catch (error) { logger.warn({ err: error }, "Rejected Cloud control assertion"); diff --git a/server/src/routes/ephemeral-enrollment.ts b/server/src/routes/ephemeral-enrollment.ts index 82d1e52a7b5..b626232c9f7 100644 --- a/server/src/routes/ephemeral-enrollment.ts +++ b/server/src/routes/ephemeral-enrollment.ts @@ -2,6 +2,7 @@ import { Router } from "express"; import { z } from "zod"; import type { Db } from "@paperclipai/db"; import { validate } from "../middleware/validate.js"; +import { verifiedCloudControlClaims } from "../middleware/cloud-control.js"; import { prepareOnlineEphemeralEnrollment } from "../services/ephemeral-enrollment.js"; export function ephemeralEnrollmentRoutes(db: Db) { @@ -11,7 +12,9 @@ export function ephemeralEnrollmentRoutes(db: Db) { if (req.actor.source !== "cloud_control") { res.status(403).json({ error: "cloud_control_required" }); return; } next(); }, validate(z.object({ attemptId: z.string().uuid() }).strict()), async (req, res) => { - res.json(await prepareOnlineEphemeralEnrollment(db, req.body.attemptId)); + const claims = verifiedCloudControlClaims(req); + if (claims?.action !== "ephemeral-enrollment:prepare") { res.status(403).json({ error: "cloud_control_required" }); return; } + res.json(await prepareOnlineEphemeralEnrollment(db, req.body.attemptId, claims)); }); return router; } diff --git a/server/src/services/activity-log.ts b/server/src/services/activity-log.ts index 4e3aab320b0..de1940acd24 100644 --- a/server/src/services/activity-log.ts +++ b/server/src/services/activity-log.ts @@ -164,10 +164,11 @@ export function publishActivity(publication: ActivityPublication) { } finally { finish(); } } -export async function persistActivity(db: Db, input: LogActivityInput) { +export async function persistActivity(db: Db, input: LogActivityInput, receipt?: { id: string }) { const redactedDetails = await redactActivityDetails(db, input.authorization ? { ...input.details, permissionDecision: input.authorization, ...(input.agentApiKeyId ? { agentApiKeyId: input.agentApiKeyId } : {}) } : input.details ?? null); const responsibleUserId = await resolveResponsibleUserIdForActivity(db, input); const insert = (runner: Db) => runner.insert(activityLog).values({ + ...(receipt ? { id: receipt.id } : {}), companyId: input.companyId, actorType: input.actorType, actorId: input.actorId, diff --git a/server/src/services/ephemeral-enrollment.test.ts b/server/src/services/ephemeral-enrollment.test.ts index 507defddde5..f638e2109cf 100644 --- a/server/src/services/ephemeral-enrollment.test.ts +++ b/server/src/services/ephemeral-enrollment.test.ts @@ -7,14 +7,16 @@ vi.mock("./ephemeral-server-contract.js", async original => ({ ...await original(), readEphemeralServerDataEligibility: owners.eligibility, })); vi.mock("./decision-signing.js", () => ({ resolveDecisionSigningSecret: owners.key })); -vi.mock("./activity-log.js", () => ({ logActivity: owners.audit })); +vi.mock("./activity-log.js", () => ({ persistActivity: owners.audit, publishActivity: vi.fn() })); vi.mock("../config.js", () => ({ loadConfig: () => ({ databaseUrl: "postgres://fixture:fixture@db.test/database", storageProvider: "s3" }) })); const db = {} as Db; const attempt = "12345678-1234-4234-8234-123456789abc"; const key = "fixture-decision-signing-key-0123456789"; +const control = { requestId: "fixture-signed-request", exp: 0 }; beforeEach(() => { vi.resetAllMocks(); + control.exp = Math.floor(Date.now() / 1000) + 60; vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-managed-instance"); vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "stack-fixture"); vi.stubEnv("PAPERCLIP_BUILD_COMMIT", "a".repeat(40)); @@ -26,17 +28,18 @@ beforeEach(() => { vi.stubEnv("PAPERCLIP_DATABASE_LISTEN_URL", "postgres://fixture:fixture@db.test/database"); owners.eligibility.mockResolvedValue({ eligible: true, blockers: [] }); owners.key.mockReturnValue(key); - owners.audit.mockResolvedValue(undefined); + owners.audit.mockResolvedValue({ publication: {} }); }); afterEach(() => vi.unstubAllEnvs()); it("exports the original key only after owner eligibility, config validation and a secret-free audit", async () => { - const result = await prepareOnlineEphemeralEnrollment(db, attempt); + const result = await prepareOnlineEphemeralEnrollment(db, attempt, control); expect(result).toMatchObject({ attemptId: attempt, stackId: "stack-fixture", commit: "a".repeat(40), deploymentId: "11111111-1111-4111-8111-111111111111", ephemeralServer: false, enrollmentReady: false, decisionSigningSecret: key }); expect(result.decisionKeyDigest).toMatch(/^[a-f0-9]{64}$/); expect(owners.key).toHaveBeenCalledWith({ allowCreate: false }); + expect(owners.eligibility).toHaveBeenCalledWith(db, { purpose: "startup" }); expect(owners.audit).toHaveBeenCalledOnce(); expect(JSON.stringify(owners.audit.mock.calls)).not.toContain(key); expect(process.env.PAPERCLIP_EPHEMERAL_SERVER).toBe("false"); @@ -44,29 +47,35 @@ it("exports the original key only after owner eligibility, config validation and it("does not read or export the key while a local obligation remains", async () => { owners.eligibility.mockResolvedValue({ eligible: false, blockers: ["workspaces_require_migration"] }); - await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toMatchObject({ status: 409, + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toMatchObject({ status: 409, details: { blockers: ["workspaces_require_migration"] } }); expect(owners.key).not.toHaveBeenCalled(); expect(owners.audit).not.toHaveBeenCalled(); }); it("refuses an invalid replacement configuration without publishing or auditing a key", async () => { vi.stubEnv("PAPERCLIP_DATABASE_LISTEN_URL", "postgres://fixture:fixture@other.test/database"); - await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toThrow("direct connection to the same database"); + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toThrow("direct connection to the same database"); expect(owners.audit).not.toHaveBeenCalled(); }); it("fails closed if key preservation auditing fails", async () => { owners.audit.mockRejectedValue(new Error("fixture_audit_failed")); - await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toThrow("fixture_audit_failed"); + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toThrow("fixture_audit_failed"); }); it("identifies the armed deployment without granting detachment authority", async () => { vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); - expect(await prepareOnlineEphemeralEnrollment(db, attempt)).toMatchObject({ ephemeralServer: true, enrollmentReady: false }); + expect(await prepareOnlineEphemeralEnrollment(db, attempt, control)).toMatchObject({ ephemeralServer: true, enrollmentReady: false }); }); it("refuses an unknown deployment before inspecting data or keys", async () => { vi.stubEnv("RAILWAY_DEPLOYMENT_ID", ""); - await expect(prepareOnlineEphemeralEnrollment(db, attempt)).rejects.toMatchObject({ status: 409 }); + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toMatchObject({ status: 409 }); expect(owners.eligibility).not.toHaveBeenCalled(); expect(owners.key).not.toHaveBeenCalled(); }); + +it("refuses an expired assertion after inspection without exporting", async () => { + control.exp = Math.floor(Date.now() / 1000) - 1; + await expect(prepareOnlineEphemeralEnrollment(db, attempt, control)).rejects.toMatchObject({ status: 401 }); + expect(owners.audit).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/ephemeral-enrollment.ts b/server/src/services/ephemeral-enrollment.ts index 95e0a210be2..415fdad5243 100644 --- a/server/src/services/ephemeral-enrollment.ts +++ b/server/src/services/ephemeral-enrollment.ts @@ -2,20 +2,21 @@ import { createHash } from "node:crypto"; import type { Db } from "@paperclipai/db"; import { readBuildCommit } from "../build-commit.js"; import { loadConfig } from "../config.js"; -import { conflict } from "../errors.js"; -import { logActivity } from "./activity-log.js"; +import { conflict, unauthorized } from "../errors.js"; +import { persistActivity, publishActivity } from "./activity-log.js"; +import type { CloudControlClaims } from "./cloud-runtime-identity.js"; import { cloudTenantPrimaryCompanyId, getCloudStackContext } from "./cloud-instance.js"; import { resolveDecisionSigningSecret } from "./decision-signing.js"; import { assertEphemeralServerConfig, ephemeralServerEnabled, readEphemeralServerDataEligibility } from "./ephemeral-server-contract.js"; -export async function prepareOnlineEphemeralEnrollment(db: Db, attemptId: string) { +export async function prepareOnlineEphemeralEnrollment(db: Db, attemptId: string, control: Pick) { const stackId = getCloudStackContext()?.stackId; const commit = readBuildCommit(); const deploymentId = process.env.RAILWAY_DEPLOYMENT_ID; if (!stackId || !commit || !deploymentId || !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(deploymentId)) { throw conflict("Online enrollment requires a stamped managed Railway deployment"); } - const eligibility = await readEphemeralServerDataEligibility(db); + const eligibility = await readEphemeralServerDataEligibility(db, { purpose: "startup" }); if (!eligibility.eligible) throw conflict("Durable data migration is incomplete", { blockers: eligibility.blockers }); const decisionSigningSecret = resolveDecisionSigningSecret({ allowCreate: false }); assertEphemeralServerConfig(loadConfig(), { ...process.env, PAPERCLIP_EPHEMERAL_SERVER: "true", @@ -27,7 +28,20 @@ export async function prepareOnlineEphemeralEnrollment(db: Db, attemptId: string decisionKeyDigest: createHash("sha256").update(decisionSigningSecret).digest("hex"), enrollmentReady: false as const, }; - await logActivity(db, { companyId: cloudTenantPrimaryCompanyId(stackId), actorType: "system", actorId: "paperclip-cloud", - action: "instance.ephemeral_enrollment.prepared", entityType: "instance", entityId: stackId, details: receipt }); + if (control.exp * 1000 <= Date.now()) throw unauthorized("Cloud preparation assertion expired"); + const bytes = createHash("sha256").update(`paperclip-ephemeral-preparation:v1:${stackId}:${control.requestId}`).digest().subarray(0, 16); + bytes[6] = (bytes[6]! & 0x0f) | 0x50; bytes[8] = (bytes[8]! & 0x3f) | 0x80; + const hex = bytes.toString("hex"); + const id = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; + try { + const { publication } = await persistActivity(db, { companyId: cloudTenantPrimaryCompanyId(stackId), actorType: "system", actorId: "paperclip-cloud", + action: "instance.ephemeral_enrollment.prepared", entityType: "instance", entityId: stackId, details: receipt }, { id }); + publishActivity(publication); + } catch (error) { + const pg = (error as { cause?: { code?: string; constraint_name?: string } }).cause ?? error as { code?: string; constraint_name?: string }; + if (pg.code === "23505" && pg.constraint_name === "activity_log_pkey") throw conflict("Cloud preparation assertion was already used"); + throw error; + } + if (control.exp * 1000 <= Date.now()) throw unauthorized("Cloud preparation assertion expired"); return { ...receipt, decisionSigningSecret }; }