diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.ts index 08123e23a55..3dde16f8fde 100644 --- a/server/src/__tests__/instance-settings-routes.test.ts +++ b/server/src/__tests__/instance-settings-routes.test.ts @@ -1,3 +1,4 @@ +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; import express from "express"; import request from "supertest"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -81,21 +82,24 @@ function localAdminActor() { describe("instance settings routes", () => { const routeModules = hoistModuleGraph(registerModuleMocks, async () => { - const [{ errorHandler }, { instanceSettingsRoutes }] = await Promise.all([ + const [{ errorHandler }, { instanceSettingsRoutes }, { cloudControlMiddleware }, controlIdentity] = await Promise.all([ vi.importActual("../middleware/index.js"), vi.importActual("../routes/instance-settings.js"), + vi.importActual("../middleware/cloud-control.js"), + vi.importActual("../services/cloud-runtime-identity.js"), ]); - return { errorHandler, instanceSettingsRoutes }; + return { errorHandler, instanceSettingsRoutes, cloudControlMiddleware, controlIdentity }; }); - function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise) { - const { errorHandler, instanceSettingsRoutes } = routeModules.value; + function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise, verifyControl = false) { + const { errorHandler, instanceSettingsRoutes, cloudControlMiddleware } = routeModules.value; const app = express(); app.use(express.json()); app.use((req, _res, next) => { req.actor = actor; next(); }); + if (verifyControl) app.use(cloudControlMiddleware()); app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers, prepareBackup)); app.use(errorHandler); return app; @@ -1101,7 +1105,7 @@ describe("instance settings routes", () => { await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200); expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 }); await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200); - expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false, false); + expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined }); }); it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => { @@ -1128,7 +1132,7 @@ describe("instance settings routes", () => { const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1"); expect(res.status).toBe(200); expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report }); - expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false, false); + expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined }); expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus); }); @@ -1142,7 +1146,7 @@ describe("instance settings routes", () => { .get("/api/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&ownerId=fixture-owner").expect(200); expect(response.body.idleSleepSafety.diagnostics.checks).toEqual([{ reason: "agent_api_keys", state: "present" }]); expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, - "fixture-owner", expect.any(Function), undefined, undefined, false, false, { collectDiagnostics: true }); + "fixture-owner", expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined, collectDiagnostics: true }); mockReadIdleSleepSafety.mockClear(); for (const actor of [nonAdminActor, { type: "agent", agentId: "agent-1", companyId: "company-1", source: "api_key" }]) { await request(createApp(actor)) @@ -1164,7 +1168,93 @@ describe("instance settings routes", () => { await request(createApp({ ...adminActor, source: "cloud_control" })) .get(`/api/instance/task-drain?idleSleepSafety=1&scheduledWake=${version}&ownerId=fixture-owner`).expect(200); expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, - "fixture-owner", expect.any(Function), undefined, undefined, true, version === "2"); + "fixture-owner", expect.any(Function), undefined, undefined, true, version === "2", { incomingWake: undefined, deadlineCapability: undefined }); + }); + + describe("signed incoming capability handoff", () => { + const createControlApp = (actor: any) => createApp(actor, undefined, undefined, true); + const keyPair = generateKeyPairSync("ed25519"); + const publicJwk = { ...keyPair.publicKey.export({ format: "jwk" }), kid: "idle-route-test", alg: "EdDSA", use: "sig" }; + const ownerId = "d0b833f4-4098-42de-8420-1907f3aa4895"; + const catalog = { + incomingWake: { version: 1, ownerId, generation: "a".repeat(64), routes: ["tenant-api-http-v1"] }, + deadlineCatalog: "domain_deadlines_v1", + }; + const path = `/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`; + function assertion(claims: Record = {}) { + const identity = routeModules.value.controlIdentity; + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url"); + const iat = Math.floor(Date.now() / 1000); + const header = encode({ alg: "EdDSA", typ: identity.CLOUD_CONTROL_JWS_TYPE, kid: publicJwk.kid }); + const payload = encode({ v: 1, iss: identity.CLOUD_RUNTIME_IDENTITY_ISSUER, aud: identity.CLOUD_CONTROL_AUDIENCE, + sub: "stack-idle-route-test", action: "task-drain:read", requestId: randomUUID(), iat, exp: iat + 60, + idleCapabilities: catalog, ...claims }); + return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, "ascii"), keyPair.privateKey).toString("base64url")}`; + } + beforeEach(() => { + vi.stubEnv("PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS", JSON.stringify({ keys: [publicJwk] })); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "stack-idle-route-test"); + routeModules.value.controlIdentity.resetCloudControlReplayFenceForTests(); + mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus); + mockReadIdleSleepSafety.mockResolvedValue({ version: 3, backgroundWork: "none" }); + }); + afterEach(() => vi.unstubAllEnvs()); + + it("passes only the immutable signed catalog through real middleware and the production route", async () => { + const token = assertion(); + const response = await request(createControlApp({ type: "none" })) + .get(`${path}&incomingWake=${encodeURIComponent(JSON.stringify({ ...catalog.incomingWake, generation: "b".repeat(64) }))}`) + .set("x-paperclip-cloud-control", token) + .send({ idleCapabilities: { incomingWake: { ...catalog.incomingWake, ownerId: randomUUID() } } }).expect(200); + expect(response.body.idleSleepSafety).toEqual({ version: 3, backgroundWork: "none" }); + expect(mockReadIdleSleepSafety).toHaveBeenCalledExactlyOnceWith(mockDb, expect.any(Function), Date.now, + ownerId, expect.any(Function), undefined, undefined, false, false, + { incomingWake: catalog, deadlineCapability: catalog }); + const options = mockReadIdleSleepSafety.mock.calls[0][9]; + expect(options.incomingWake).toBe(options.deadlineCapability); + expect(Object.isFrozen(options.incomingWake)).toBe(true); + expect(Object.isFrozen(options.incomingWake.incomingWake)).toBe(true); + expect(Object.isFrozen(options.incomingWake.incomingWake.routes)).toBe(true); + await request(createControlApp({ type: "none" })).get(path).set("x-paperclip-cloud-control", token).expect(401); + expect(mockReadIdleSleepSafety).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["wrong stack", () => ({ sub: "other-stack" })], + ["wrong action", () => ({ action: "task-drain:start" })], + ["expired", () => ({ iat: Math.floor(Date.now() / 1000) - 600, exp: Math.floor(Date.now() / 1000) - 300 })], + ["unsupported catalog", () => ({ idleCapabilities: { incomingWake: { ...catalog.incomingWake, routes: ["unverified-path"] } } })], + ])("rejects %s before safety inspection", async (_name, claims) => { + await request(createControlApp(adminActor)).get(path).set("x-paperclip-cloud-control", assertion(claims())).expect(401); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + }); + + it("rejects a tampered catalog signature before safety inspection", async () => { + const [header, payload, signature] = assertion().split("."); + const changed = JSON.parse(Buffer.from(payload!, "base64url").toString()); + changed.idleCapabilities.incomingWake.generation = "b".repeat(64); + const tampered = `${header}.${Buffer.from(JSON.stringify(changed)).toString("base64url")}.${signature}`; + await request(createControlApp(adminActor)).get(path).set("x-paperclip-cloud-control", tampered).expect(401); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + }); + + it("leaves legacy signed reads without a catalog conservative", async () => { + await request(createControlApp({ type: "none" })).get(path) + .set("x-paperclip-cloud-control", assertion({ idleCapabilities: undefined })).expect(200); + expect(mockReadIdleSleepSafety.mock.calls[0][9]).toEqual({ incomingWake: undefined, deadlineCapability: undefined }); + }); + + it("cannot install capabilities from tenant, actor, query or body values", async () => { + for (const source of ["session", "cloud_tenant", "local_implicit", "cloud_control"]) { + const actor = { ...adminActor, source, idleCapabilities: catalog, incomingWake: catalog }; + await request(createControlApp(actor)).get(`${path}&incomingWake=${encodeURIComponent(JSON.stringify(catalog))}`) + .send({ idleCapabilities: catalog }).expect(200); + expect(mockReadIdleSleepSafety.mock.lastCall?.[9]).toEqual({ incomingWake: undefined, deadlineCapability: undefined }); + } + mockReadIdleSleepSafety.mockClear(); + await request(createControlApp(nonAdminActor)).get(path).send({ idleCapabilities: catalog }).expect(403); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + }); }); it("uses the live worker manager for inspection and releases only the owned hold", async () => { @@ -1199,7 +1289,7 @@ describe("instance settings routes", () => { expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200); expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, - "fixture-owner", expect.any(Function), undefined, backup, false, false); + "fixture-owner", expect.any(Function), undefined, backup, false, false, { incomingWake: undefined, deadlineCapability: undefined }); mockReadIdleSleepSafety.mockClear(); await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403); expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); @@ -1221,7 +1311,7 @@ describe("instance settings routes", () => { const control = createApp({ ...adminActor, source: "cloud_control" }, undefined, backup); await request(control).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200); expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, - "fixture-owner", expect.any(Function), undefined, backup, false, false); + "fixture-owner", expect.any(Function), undefined, backup, false, false, { incomingWake: undefined, deadlineCapability: undefined }); } finally { delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN; }