diff --git a/doc/CUSTOMER-SUCCESS-INSPECTION.md b/doc/CUSTOMER-SUCCESS-INSPECTION.md index e20a2c807f3..bd27a2ad95c 100644 --- a/doc/CUSTOMER-SUCCESS-INSPECTION.md +++ b/doc/CUSTOMER-SUCCESS-INSPECTION.md @@ -1,7 +1,7 @@ # Cloud customer-success inspection support -Paperclip exposes `/api/customer-success/v1/run-authority` and `/read` for the -coordinated Cloud inspection broker. This foundation supports one registered +Paperclip exposes `/api/customer-success/v1/run-authority`, `/run-liveness` and +`/read` for the coordinated Cloud inspection broker. This foundation supports one registered Paperclip agent; bot creation, scheduling, scoring and reporting are separate. The authority endpoint is disabled unless @@ -12,6 +12,20 @@ paused, cancelled/stopped, finished and unsupported-runtime agents are rejected. No identity GET provisions keys. Normal API JWT compatibility remains unchanged; legacy signatures are rejected specifically at this authority boundary. +The run liveness check (`POST /run-liveness`) is also disabled unless +`PAPERCLIP_CUSTOMER_SUCCESS_AUTHORITY_ENABLED=true`. It lets Cloud ask, with its +own inspection key rather than an agent credential, whether one run of one agent +in one company is live. It refuses cookies and `Authorization` headers and +accepts only an Ed25519 JWS in the inspection header, verified against +`PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_JWKS`, with type +`paperclip-inspection-liveness+jwt`, audience +`paperclip-customer-success-liveness/v1`, this stack as subject and a window of +at most sixty seconds. The canonical JSON body `{agentId, companyId, runId}` must +equal the signed claims. The answer is `{version: 1, companyId, agentId, runId, +active}`, using the same run rules as the authority endpoint (one shared +predicate) but without requiring a provisioned identity key. It runs in a +read-only transaction and creates no actor, session, activity or key. + Tenant reads are disabled unless `PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_ENABLED=true`. Configure the public-only `PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_JWKS` and the stable HTTPS diff --git a/server/src/__tests__/customer-success-run-liveness.test.ts b/server/src/__tests__/customer-success-run-liveness.test.ts new file mode 100644 index 00000000000..1f1edb264ab --- /dev/null +++ b/server/src/__tests__/customer-success-run-liveness.test.ts @@ -0,0 +1,137 @@ +import { randomUUID, generateKeyPairSync, sign, type KeyObject } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { count, eq } from "drizzle-orm"; +import * as schema from "@paperclipai/db"; +import { canonicalJson, INSPECTION_HEADER } from "@paperclipai/shared"; +import { + customerSuccessRoutes, + RUN_LIVENESS_AUDIENCE, + RUN_LIVENESS_TYPE, + type RunLivenessClaims, +} from "../routes/customer-success.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import type { StorageService } from "../storage/types.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const PATH = "/api/customer-success/v1/run-liveness"; +const STACK = "fixture-stack"; +const storage = {} as StorageService; +const keys = generateKeyPairSync("ed25519"); +const jwks = { keys: [{ ...keys.publicKey.export({ format: "jwk" }), kid: "cloud" }] }; + +function token(claims: RunLivenessClaims, options: { privateKey?: KeyObject; typ?: string } = {}) { + const header = Buffer.from( + canonicalJson({ alg: "EdDSA", typ: options.typ ?? RUN_LIVENESS_TYPE, kid: "cloud" }), + ).toString("base64url"); + const payload = Buffer.from(canonicalJson(claims)).toString("base64url"); + const input = `${header}.${payload}`; + return `${input}.${sign(null, Buffer.from(input), options.privateKey ?? keys.privateKey).toString("base64url")}`; +} + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("Cloud run liveness check for inspection authority", () => { + let db: schema.Db; + let database: Awaited>; + let companyId: string; + let agentId: string; + let runId: string; + let finishedRunId: string; + let app: express.Express; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("run-liveness-db-"); + db = schema.createDb(database.connectionString); + const [company] = await db.insert(schema.companies).values({ name: "Customer", issuePrefix: "LIV" }).returning(); + companyId = company.id; + // A plain process agent: liveness needs no managed identity key. + const [agent] = await db.insert(schema.agents).values({ companyId, name: "Fixture agent", adapterType: "process" }).returning(); + agentId = agent.id; + const runs = await db.insert(schema.heartbeatRuns).values([ + { companyId, agentId, status: "running", startedAt: new Date() }, + { companyId, agentId, status: "succeeded", startedAt: new Date(), finishedAt: new Date() }, + ]).returning(); + runId = runs[0].id; + finishedRunId = runs[1].id; + app = express(); + app.use(express.json()); + app.use("/api/customer-success/v1", customerSuccessRoutes(db, storage)); + app.use(errorHandler); + }, 60_000); + afterAll(async () => { await database?.cleanup(); }); + beforeEach(() => { + vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_AUTHORITY_ENABLED", "true"); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", STACK); + vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_JWKS", JSON.stringify(jwks)); + }); + afterEach(() => vi.unstubAllEnvs()); + + function claims(overrides: Partial = {}): RunLivenessClaims { + const now = Math.floor(Date.now() / 1000); + return { + v: 1, iss: "paperclip-cloud", aud: RUN_LIVENESS_AUDIENCE, sub: STACK, jti: randomUUID(), + iat: now, exp: now + 60, companyId, agentId, runId, ...overrides, + }; + } + const body = (value: RunLivenessClaims) => canonicalJson({ agentId: value.agentId, companyId: value.companyId, runId: value.runId }); + const ask = (value: RunLivenessClaims, jws = token(value), payload = body(value)) => + request(app).post(PATH).set("Content-Type", "application/json").set(INSPECTION_HEADER, jws).send(payload); + const sideEffectCounts = async () => Promise.all([ + schema.activityLog, schema.agentApiKeys, schema.agentIdentityKeys, + ].map(async (table) => (await db.select({ value: count() }).from(table))[0].value)); + + it("answers active for a running run without creating any state", async () => { + const before = await sideEffectCounts(); + const question = claims(); + const response = await ask(question); + expect(response.status).toBe(200); + expect(response.headers["cache-control"]).toBe("no-store"); + expect(response.body).toEqual({ version: 1, companyId, agentId, runId, active: true }); + expect(await sideEffectCounts()).toEqual(before); + }); + + it("answers inactive for a finished or write-revoked run, a paused agent, another company or an unknown run", async () => { + expect((await ask(claims({ runId: finishedRunId }))).body).toMatchObject({ runId: finishedRunId, active: false }); + expect((await ask(claims({ runId: randomUUID() }))).body).toMatchObject({ active: false }); + expect((await ask(claims({ companyId: randomUUID() }))).body).toMatchObject({ active: false }); + await db.update(schema.heartbeatRuns).set({ resultJson: { executionCancellation: { state: "requested" } } }).where(eq(schema.heartbeatRuns.id, runId)); + try { + expect((await ask(claims())).body).toMatchObject({ active: false }); + } finally { + await db.update(schema.heartbeatRuns).set({ resultJson: null }).where(eq(schema.heartbeatRuns.id, runId)); + } + await db.update(schema.agents).set({ status: "paused" }).where(eq(schema.agents.id, agentId)); + try { + expect((await ask(claims())).body).toEqual({ version: 1, companyId, agentId, runId, active: false }); + } finally { + await db.update(schema.agents).set({ status: "idle" }).where(eq(schema.agents.id, agentId)); + } + }); + + it("rejects a question for another stack, an expired or over-long window, a bad signature and another token type", async () => { + const now = Math.floor(Date.now() / 1000); + expect((await ask(claims({ sub: "other-stack" }))).status).toBe(403); + expect((await ask(claims({ iat: now - 120, exp: now - 60 }))).status).toBe(403); + expect((await ask(claims({ iat: now, exp: now + 61 }))).status).toBe(403); + const value = claims(); + expect((await ask(value, token(value, { privateKey: generateKeyPairSync("ed25519").privateKey }))).status).toBe(403); + expect((await ask(value, token(value, { typ: "paperclip-inspection+jwt" }))).status).toBe(403); + expect((await ask(claims({ aud: "paperclip-customer-success/v1" as never }))).status).toBe(403); + }); + + it("rejects a body that differs from the signed question", async () => { + const value = claims(); + expect((await ask(value, token(value), body({ ...value, runId: finishedRunId }))).status).toBe(403); + expect((await ask(value, token(value), canonicalJson({ agentId, companyId, runId, extra: true }))).status).toBe(403); + }); + + it("is off unless run authority is enabled, and refuses browser or bearer credentials", async () => { + const value = claims(); + vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_AUTHORITY_ENABLED", "false"); + expect((await ask(value)).status).toBe(403); + vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_AUTHORITY_ENABLED", "true"); + expect((await ask(value).set("Cookie", "session=ignored")).status).toBe(403); + expect((await ask(value).set("Authorization", "Bearer ignored")).status).toBe(403); + expect((await ask(value)).status).toBe(200); + }); +}); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 3a8e0157049..5f409e5f5fb 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -137,6 +137,7 @@ const explicitOpenApiOperationCoverageExclusions = new Set([ // Ordinary board sessions and agent API keys cannot invoke these endpoints. "GET /api/customer-success/v1/run-authority", "POST /api/customer-success/v1/read", + "POST /api/customer-success/v1/run-liveness", // OAuth discovery and protocol endpoints have their own metadata contract; // browser connection-management operations remain documented in the board API. // Markdown rendering of the separately documented assistant setup page. diff --git a/server/src/routes/customer-success.ts b/server/src/routes/customer-success.ts index ba7cc37eb83..0e2109753f3 100644 --- a/server/src/routes/customer-success.ts +++ b/server/src/routes/customer-success.ts @@ -11,49 +11,93 @@ import { } from "@paperclipai/shared"; import type { StorageService } from "../storage/types.js"; import { getCloudRuntimeIdentity } from "../services/cloud-runtime-identity.js"; -import { customerSuccessRunAuthority } from "../services/customer-success-authority.js"; +import { customerSuccessRunAuthority, customerSuccessRunLiveness } from "../services/customer-success-authority.js"; import { readCustomerSuccessResource, type CustomerSuccessInspectionOptions } from "../services/customer-success-inspection.js"; import { forbidden, unprocessable } from "../errors.js"; +/** Cloud → home stack run-liveness question (see `customerSuccessRunLiveness`). */ +export const RUN_LIVENESS_TYPE = "paperclip-inspection-liveness+jwt"; +export const RUN_LIVENESS_AUDIENCE = "paperclip-customer-success-liveness/v1"; +export interface RunLivenessQuestion { + companyId: string; + agentId: string; + runId: string; +} +export interface RunLivenessClaims extends RunLivenessQuestion { + v: 1; + iss: "paperclip-cloud"; + aud: typeof RUN_LIVENESS_AUDIENCE; + sub: string; + jti: string; + iat: number; + exp: number; +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * Verifies a Cloud-signed EdDSA JWS of one type and the claims every Cloud + * question shares: version, issuer, audience, this stack as subject, and an + * unexpired window of at most 60 seconds. Throws a plain error on any doubt. + */ +function verifyCloudJws( + token: string, + keys: { keys: Array> }, + expected: { typ: string; aud: string; stackId: string }, + now: number, +): Claims { + const parts = token.split("."); + if (parts.length !== 3 || token.length > 16384) throw new Error(); + const header = JSON.parse(Buffer.from(parts[0], "base64url").toString()); + const claims = JSON.parse(Buffer.from(parts[1], "base64url").toString()) as Claims; + const jwk = keys.keys.find( + (k) => k.kid === header.kid && k.kty === "OKP" && k.crv === "Ed25519" && !k.d, + ); + if ( + header.alg !== "EdDSA" || + header.typ !== expected.typ || + !jwk || + !verify( + null, + Buffer.from(`${parts[0]}.${parts[1]}`), + createPublicKey({ key: jwk, format: "jwk" }), + Buffer.from(parts[2], "base64url"), + ) + ) + throw new Error(); + const seconds = Math.floor(now / 1000); + if ( + !claims || + claims.v !== 1 || + claims.iss !== "paperclip-cloud" || + claims.aud !== expected.aud || + claims.sub !== expected.stackId || + !Number.isInteger(claims.iat) || + !Number.isInteger(claims.exp) || + claims.exp <= seconds || + claims.iat > seconds + 5 || + claims.exp - claims.iat > 60 || + claims.exp <= claims.iat || + !claims.jti + ) + throw new Error(); + return claims; +} + export function verifyInspectionPermit( token: string, keys: { keys: Array> }, stackId: string, now = Date.now(), ): InspectionPermit { - const parts = token.split("."); - if (parts.length !== 3 || token.length > 16384) throw forbidden("Invalid inspection permit"); try { - const header = JSON.parse(Buffer.from(parts[0], "base64url").toString()); - const claims = JSON.parse(Buffer.from(parts[1], "base64url").toString()) as InspectionPermit; - const jwk = keys.keys.find( - (k) => k.kid === header.kid && k.kty === "OKP" && k.crv === "Ed25519" && !k.d, + const claims = verifyCloudJws( + token, + keys, + { typ: INSPECTION_PERMIT_TYPE, aud: INSPECTION_AUDIENCE, stackId }, + now, ); if ( - header.alg !== "EdDSA" || - header.typ !== INSPECTION_PERMIT_TYPE || - !jwk || - !verify( - null, - Buffer.from(`${parts[0]}.${parts[1]}`), - createPublicKey({ key: jwk, format: "jwk" }), - Buffer.from(parts[2], "base64url"), - ) - ) - throw new Error(); - const seconds = Math.floor(now / 1000); - if ( - claims.v !== 1 || - claims.iss !== "paperclip-cloud" || - claims.aud !== INSPECTION_AUDIENCE || - claims.sub !== stackId || - !Number.isInteger(claims.iat) || - !Number.isInteger(claims.exp) || - claims.exp <= seconds || - claims.iat > seconds + 5 || - claims.exp - claims.iat > 60 || - claims.exp <= claims.iat || - !claims.jti || !claims.grantId || !claims.requestId || !claims.agentId || @@ -69,6 +113,27 @@ export function verifyInspectionPermit( } } +export function verifyRunLivenessQuestion( + token: string, + keys: { keys: Array> }, + stackId: string, + now = Date.now(), +): RunLivenessClaims { + try { + const claims = verifyCloudJws( + token, + keys, + { typ: RUN_LIVENESS_TYPE, aud: RUN_LIVENESS_AUDIENCE, stackId }, + now, + ); + if (![claims.companyId, claims.agentId, claims.runId].every((id) => typeof id === "string" && UUID.test(id))) + throw new Error(); + return claims; + } catch { + throw forbidden("Invalid run liveness question"); + } +} + /** Mounted before actor middleware: none of these reads creates tenant state. */ export function customerSuccessRoutes(db: Db, storage: StorageService, options: CustomerSuccessInspectionOptions = {}) { const router = Router(); @@ -128,6 +193,34 @@ export function customerSuccessRoutes(db: Db, storage: StorageService, options: throw unprocessable("Inspection response is too large; use pagination or a byte range"); res.type("application/json").send(body); }); + // Cloud run liveness check for inspection authority. Cloud asks with its + // own signed, stack-addressed, 60-second question; no agent credential, + // identity key, actor, session or activity is involved. + router.post("/run-liveness", async (req, res) => { + if (process.env.PAPERCLIP_CUSTOMER_SUCCESS_AUTHORITY_ENABLED !== "true") + throw forbidden("Run authority is disabled"); + if (req.headers.authorization || req.headers.cookie) + throw forbidden("Only a Cloud run liveness question is accepted"); + const stackId = getCloudRuntimeIdentity()?.stackId ?? process.env.PAPERCLIP_CLOUD_STACK_ID; + const rawKeys = process.env.PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_JWKS; + const token = req.get(INSPECTION_HEADER); + if (!stackId || !rawKeys || !token) throw forbidden("Inspection trust is unavailable"); + let keys: { keys: Array> }; + try { + keys = JSON.parse(rawKeys); + } catch { + throw forbidden("Inspection trust is invalid"); + } + const claims = verifyRunLivenessQuestion(token, keys, stackId); + const question: RunLivenessQuestion = { + companyId: claims.companyId, + agentId: claims.agentId, + runId: claims.runId, + }; + if (canonicalJson(req.body ?? null) !== canonicalJson(question)) + throw forbidden("Run liveness parameters do not match the question"); + res.json({ version: 1, ...question, active: await customerSuccessRunLiveness(db, question) }); + }); // Unknown methods/paths terminate here rather than entering normal auth. router.use((_req, res) => { res.status(404).json({ error: "Unsupported inspection endpoint" }); diff --git a/server/src/services/customer-success-authority.ts b/server/src/services/customer-success-authority.ts index 2081902ae08..1b37558cd76 100644 --- a/server/src/services/customer-success-authority.ts +++ b/server/src/services/customer-success-authority.ts @@ -6,6 +6,24 @@ import { agentRunWritesRevoked } from "../agent-run-cancellation.js"; import { forbidden } from "../errors.js"; import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-identity.js"; +/** + * The one rule for "this agent's run is live", shared by run authority and + * the Cloud run liveness check so the two can never disagree. + */ +function isActiveAgentRun( + agent: { status: string } | undefined, + run: (typeof heartbeatRuns.$inferSelect) | undefined, +): run is typeof heartbeatRuns.$inferSelect { + return Boolean( + agent && + ["idle", "running", "active"].includes(agent.status) && + run && + run.status === "running" && + !run.finishedAt && + !agentRunWritesRevoked(run), + ); +} + /** No actor/session resolution, key provisioning, or run identity capture. */ export async function customerSuccessRunAuthority(db: Db, bearer: string): Promise { const claims = verifyLocalAgentJwt(bearer, { strictRunAuthority: true }); @@ -29,11 +47,7 @@ export async function customerSuccessRunAuthority(db: Db, bearer: string): Promi if ( !agent || !supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run?.driverKind) || - !["idle", "running", "active"].includes(agent.status) || - !run || - run.status !== "running" || - run.finishedAt || - agentRunWritesRevoked(run) + !isActiveAgentRun(agent, run) ) throw forbidden("The managed run is not active"); const identity = await agentIdentityService(tx as unknown as Db).getPublicIdentity( @@ -54,3 +68,36 @@ export async function customerSuccessRunAuthority(db: Db, bearer: string): Promi { accessMode: "read only" }, ); } + +/** + * Cloud run liveness check for inspection authority: is this run of this + * agent in this company active right now? It uses the run rules of + * `customerSuccessRunAuthority` but needs no agent credential or managed + * identity key, because Cloud asks with its own signed question. Read-only; + * it creates no actor, session, activity or key. + */ +export async function customerSuccessRunLiveness( + db: Db, + question: { companyId: string; agentId: string; runId: string }, +): Promise { + return db.transaction( + async (tx) => { + const [agent] = await tx + .select({ status: agents.status }) + .from(agents) + .where(and(eq(agents.id, question.agentId), eq(agents.companyId, question.companyId))); + const [run] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, question.runId), + eq(heartbeatRuns.agentId, question.agentId), + eq(heartbeatRuns.companyId, question.companyId), + ), + ); + return isActiveAgentRun(agent, run); + }, + { accessMode: "read only" }, + ); +}