mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
Merge remote-tracking branch 'origin/master' into fix/runner-paid-matrix-integrity
* origin/master: ci(runner): prepare target lockfile once for paid validation (#12774) chore(deps): bump paperclipai/paperclip/.github/workflows/pr-trusted.yml from39b8ee2960tof038633bf5(#12562) chore(deps): bump sharp from 0.35.3 to 0.35.4 (#12563) chore(deps): bump @mdxeditor/editor from 4.2.1 to 4.2.3 (#12564) fix(server): stop paging Sentry for supervised boot races in managed cloud (#12772) Secure Cloud canonical runtime identity (#12766) # Conflicts: # .github/workflows/runner-full-stack-e2e.yml # tests/runner-e2e/README.md # tests/runner-e2e/SECURITY.md # tests/runner-e2e/workflow-security.test.ts
This commit is contained in:
commit
9e5f0e60fa
25 files changed
+1594
-211
No files matched your search
@@ -0,0 +1,261 @@
|
||||
import { generateKeyPairSync, sign } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createDb, instanceSettings } from "@paperclipai/db";
|
||||
import {
|
||||
applyCloudRuntimeIdentityAssertion,
|
||||
CLOUD_RUNTIME_IDENTITY_AUDIENCE,
|
||||
CLOUD_RUNTIME_IDENTITY_ISSUER,
|
||||
CLOUD_RUNTIME_IDENTITY_JWS_TYPE,
|
||||
getCloudRuntimeIdentity,
|
||||
initializeCloudRuntimeIdentity,
|
||||
resetCloudRuntimeIdentityForTests,
|
||||
runtimePublicOrigin,
|
||||
} from "../services/cloud-runtime-identity.js";
|
||||
import { routineWebhookUrl } from "../services/routines.js";
|
||||
import { paperclipCloudConnectorEnrollmentStatus } from "../services/paperclip-cloud-connector-enrollment.js";
|
||||
import { cloudRuntimeIdentityMiddleware } from "../middleware/cloud-runtime-identity.js";
|
||||
import { healthRoutes } from "../routes/health.js";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
const STACK_ID = "stack-pool-123";
|
||||
const POOL_ORIGIN = "https://pool-123.staging.paperclip.app";
|
||||
const CANONICAL_ORIGIN = "https://gonzo.staging.paperclip.app";
|
||||
const NOW = new Date("2099-01-01T00:00:00.000Z");
|
||||
|
||||
const pair = generateKeyPairSync("ed25519");
|
||||
const publicJwk = {
|
||||
...pair.publicKey.export({ format: "jwk" }),
|
||||
kid: "runtime-identity-test-key",
|
||||
use: "sig",
|
||||
alg: "EdDSA",
|
||||
};
|
||||
|
||||
function encodeJson(value: Record<string, unknown>) {
|
||||
return Buffer.from(JSON.stringify(value)).toString("base64url");
|
||||
}
|
||||
|
||||
function assertion(input: {
|
||||
claims?: Record<string, unknown>;
|
||||
header?: Record<string, unknown>;
|
||||
signingKey?: typeof pair.privateKey;
|
||||
} = {}) {
|
||||
const iat = Math.floor(NOW.getTime() / 1000);
|
||||
const header = encodeJson({
|
||||
alg: "EdDSA",
|
||||
typ: CLOUD_RUNTIME_IDENTITY_JWS_TYPE,
|
||||
kid: publicJwk.kid,
|
||||
...input.header,
|
||||
});
|
||||
const payload = encodeJson({
|
||||
v: 1,
|
||||
iss: CLOUD_RUNTIME_IDENTITY_ISSUER,
|
||||
aud: CLOUD_RUNTIME_IDENTITY_AUDIENCE,
|
||||
sub: STACK_ID,
|
||||
claimId: "pool-entry-123",
|
||||
previousOrigin: POOL_ORIGIN,
|
||||
canonicalOrigin: CANONICAL_ORIGIN,
|
||||
stackSlug: "gonzo",
|
||||
iat,
|
||||
exp: iat + 300,
|
||||
...input.claims,
|
||||
});
|
||||
const signature = sign(
|
||||
null,
|
||||
Buffer.from(`${header}.${payload}`, "ascii"),
|
||||
input.signingKey ?? pair.privateKey,
|
||||
).toString("base64url");
|
||||
return `${header}.${payload}.${signature}`;
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("Cloud runtime identity", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
const originalEnv = { ...process.env };
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-cloud-runtime-identity-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await db.delete(instanceSettings);
|
||||
resetCloudRuntimeIdentityForTests();
|
||||
process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "test-tenant-server-token";
|
||||
process.env.PAPERCLIP_CLOUD_STACK_ID = STACK_ID;
|
||||
process.env.PAPERCLIP_CLOUD_API_ORIGIN = "https://my-staging.paperclip.app";
|
||||
process.env.PAPERCLIP_PUBLIC_URL = POOL_ORIGIN;
|
||||
process.env.PAPERCLIP_AUTH_PUBLIC_BASE_URL = POOL_ORIGIN;
|
||||
process.env.PAPERCLIP_API_URL = POOL_ORIGIN;
|
||||
process.env.PAPERCLIP_PRIMARY_HOST = "pool-123.staging.paperclip.app";
|
||||
process.env.PAPERCLIP_STACK_SLUG = "pool-123";
|
||||
process.env.PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS = JSON.stringify({ keys: [publicJwk] });
|
||||
process.env.PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID = "managed-instance";
|
||||
process.env.PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY = "managed-signing-key";
|
||||
process.env.PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY = "managed-sealing-key";
|
||||
process.env.PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT = "staging";
|
||||
process.env.PAPERCLIP_CLOUD_CONNECTOR_BASE_URL = "https://my-staging.paperclip.app";
|
||||
await initializeCloudRuntimeIdentity(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of [
|
||||
"PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN",
|
||||
"PAPERCLIP_CLOUD_STACK_ID",
|
||||
"PAPERCLIP_CLOUD_API_ORIGIN",
|
||||
"PAPERCLIP_PUBLIC_URL",
|
||||
"PAPERCLIP_AUTH_PUBLIC_BASE_URL",
|
||||
"PAPERCLIP_API_URL",
|
||||
"PAPERCLIP_PRIMARY_HOST",
|
||||
"PAPERCLIP_STACK_SLUG",
|
||||
"PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS",
|
||||
"PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID",
|
||||
"PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY",
|
||||
"PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY",
|
||||
"PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT",
|
||||
"PAPERCLIP_CLOUD_CONNECTOR_BASE_URL",
|
||||
"PAPERCLIP_RUNTIME_API_CANDIDATES_JSON",
|
||||
]) {
|
||||
const original = originalEnv[key];
|
||||
if (original === undefined) delete process.env[key];
|
||||
else process.env[key] = original;
|
||||
}
|
||||
resetCloudRuntimeIdentityForTests();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
it("persists and immediately applies a valid one-time claim", async () => {
|
||||
const applied = await applyCloudRuntimeIdentityAssertion({
|
||||
db,
|
||||
compactJws: assertion(),
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(applied.canonicalOrigin).toBe(CANONICAL_ORIGIN);
|
||||
expect(getCloudRuntimeIdentity()?.stackSlug).toBe("gonzo");
|
||||
expect(runtimePublicOrigin()).toBe(CANONICAL_ORIGIN);
|
||||
expect(process.env.PAPERCLIP_PUBLIC_URL).toBe(CANONICAL_ORIGIN);
|
||||
expect(process.env.PAPERCLIP_AUTH_PUBLIC_BASE_URL).toBe(CANONICAL_ORIGIN);
|
||||
expect(process.env.PAPERCLIP_API_URL).toBe(CANONICAL_ORIGIN);
|
||||
expect(process.env.PAPERCLIP_PRIMARY_HOST).toBe("gonzo.staging.paperclip.app");
|
||||
expect(process.env.PAPERCLIP_STACK_SLUG).toBe("gonzo");
|
||||
expect(routineWebhookUrl("hook-1")).toBe(
|
||||
"https://gonzo.staging.paperclip.app/api/routine-triggers/public/hook-1/fire",
|
||||
);
|
||||
expect(paperclipCloudConnectorEnrollmentStatus()).toMatchObject({
|
||||
configured: true,
|
||||
status: "active",
|
||||
origins: [CANONICAL_ORIGIN],
|
||||
});
|
||||
});
|
||||
|
||||
it("applies the assertion on the existing health request and acknowledges the exact origin", async () => {
|
||||
const requestTime = Math.floor(Date.now() / 1000);
|
||||
const app = express();
|
||||
app.use(cloudRuntimeIdentityMiddleware(db));
|
||||
app.use("/api/health", healthRoutes(db, {
|
||||
deploymentMode: "authenticated",
|
||||
deploymentExposure: "public",
|
||||
authReady: true,
|
||||
companyDeletionEnabled: false,
|
||||
}));
|
||||
|
||||
const response = await request(app)
|
||||
.get("/api/health")
|
||||
.set("x-paperclip-cloud-runtime-identity", assertion({
|
||||
claims: { iat: requestTime, exp: requestTime + 300 },
|
||||
}));
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.cloud.runtimeIdentity).toEqual({
|
||||
canonicalOrigin: CANONICAL_ORIGIN,
|
||||
stackSlug: "gonzo",
|
||||
});
|
||||
});
|
||||
|
||||
it("restores the canonical identity before consumers read stale startup variables", async () => {
|
||||
await applyCloudRuntimeIdentityAssertion({ db, compactJws: assertion(), now: NOW });
|
||||
resetCloudRuntimeIdentityForTests();
|
||||
process.env.PAPERCLIP_PUBLIC_URL = POOL_ORIGIN;
|
||||
process.env.PAPERCLIP_AUTH_PUBLIC_BASE_URL = POOL_ORIGIN;
|
||||
process.env.PAPERCLIP_API_URL = POOL_ORIGIN;
|
||||
|
||||
await initializeCloudRuntimeIdentity(db);
|
||||
|
||||
expect(runtimePublicOrigin()).toBe(CANONICAL_ORIGIN);
|
||||
expect(process.env.PAPERCLIP_API_URL).toBe(CANONICAL_ORIGIN);
|
||||
});
|
||||
|
||||
it("accepts the identical claim after a restart with already-aligned provider variables", async () => {
|
||||
await applyCloudRuntimeIdentityAssertion({ db, compactJws: assertion(), now: NOW });
|
||||
resetCloudRuntimeIdentityForTests();
|
||||
process.env.PAPERCLIP_PUBLIC_URL = CANONICAL_ORIGIN;
|
||||
process.env.PAPERCLIP_AUTH_PUBLIC_BASE_URL = CANONICAL_ORIGIN;
|
||||
process.env.PAPERCLIP_API_URL = CANONICAL_ORIGIN;
|
||||
await initializeCloudRuntimeIdentity(db);
|
||||
|
||||
await expect(applyCloudRuntimeIdentityAssertion({
|
||||
db,
|
||||
compactJws: assertion(),
|
||||
now: NOW,
|
||||
})).resolves.toMatchObject({ canonicalOrigin: CANONICAL_ORIGIN });
|
||||
});
|
||||
|
||||
it("accepts an identical replay but rejects a different claim or destination", async () => {
|
||||
await applyCloudRuntimeIdentityAssertion({ db, compactJws: assertion(), now: NOW });
|
||||
await expect(applyCloudRuntimeIdentityAssertion({
|
||||
db,
|
||||
compactJws: assertion(),
|
||||
now: NOW,
|
||||
})).resolves.toMatchObject({ canonicalOrigin: CANONICAL_ORIGIN });
|
||||
await expect(applyCloudRuntimeIdentityAssertion({
|
||||
db,
|
||||
compactJws: assertion({ claims: { claimId: "another-claim" } }),
|
||||
now: NOW,
|
||||
})).rejects.toThrow("already claimed");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["expired", { exp: Math.floor(NOW.getTime() / 1000) - 1 }],
|
||||
["cross-stack", { sub: "stack-someone-else" }],
|
||||
["wrong previous origin", { previousOrigin: "https://another.staging.paperclip.app" }],
|
||||
["path-bearing destination", { canonicalOrigin: `${CANONICAL_ORIGIN}/GON` }],
|
||||
["slug mismatch", { stackSlug: "kermit" }],
|
||||
])("rejects %s assertions", async (_label, claims) => {
|
||||
await expect(applyCloudRuntimeIdentityAssertion({
|
||||
db,
|
||||
compactJws: assertion({ claims }),
|
||||
now: NOW,
|
||||
})).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("rejects unknown keys and altered signed destinations", async () => {
|
||||
await expect(applyCloudRuntimeIdentityAssertion({
|
||||
db,
|
||||
compactJws: assertion({ header: { kid: "unknown" } }),
|
||||
now: NOW,
|
||||
})).rejects.toThrow("unknown signing key");
|
||||
|
||||
const valid = assertion();
|
||||
const [header, payload, signature] = valid.split(".");
|
||||
const altered = encodeJson({
|
||||
...JSON.parse(Buffer.from(payload!, "base64url").toString("utf8")),
|
||||
canonicalOrigin: "https://attacker.example.com",
|
||||
});
|
||||
await expect(applyCloudRuntimeIdentityAssertion({
|
||||
db,
|
||||
compactJws: `${header}.${altered}.${signature}`,
|
||||
now: NOW,
|
||||
})).rejects.toThrow("signature is invalid");
|
||||
});
|
||||
});
|
||||
@@ -2146,6 +2146,18 @@ describeEmbeddedPostgres("generic remote MCP connections", () => {
|
||||
expect(JSON.stringify(response.body)).not.toContain(company.id);
|
||||
});
|
||||
|
||||
it("uses the configured auth origin for self-hosted OAuth callbacks", async () => {
|
||||
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://public.paperclip.example");
|
||||
vi.stubEnv("PAPERCLIP_AUTH_PUBLIC_BASE_URL", "https://auth.paperclip.example");
|
||||
const app = createRouteApp(db);
|
||||
|
||||
const response = await request(app).get("/api/tools/oauth/client-metadata").expect(200);
|
||||
|
||||
expect(response.body.redirect_uris).toEqual([
|
||||
"https://auth.paperclip.example/api/tools/oauth/callback",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses the managed runtime origin when no explicit callback origin is configured", async () => {
|
||||
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "");
|
||||
vi.stubEnv("PAPERCLIP_AUTH_PUBLIC_BASE_URL", "");
|
||||
|
||||
@@ -39,6 +39,7 @@ import { secretService } from "../services/secrets.ts";
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
const originalSecretsProviderEnv = process.env.PAPERCLIP_SECRETS_PROVIDER;
|
||||
const originalPaperclipApiUrlEnv = process.env.PAPERCLIP_API_URL;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
@@ -51,6 +52,7 @@ describeEmbeddedPostgres("routine service live-execution coalescing", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.PAPERCLIP_API_URL = "http://localhost:3100";
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-routines-service-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
@@ -86,6 +88,11 @@ describeEmbeddedPostgres("routine service live-execution coalescing", () => {
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
if (originalPaperclipApiUrlEnv === undefined) {
|
||||
delete process.env.PAPERCLIP_API_URL;
|
||||
} else {
|
||||
process.env.PAPERCLIP_API_URL = originalPaperclipApiUrlEnv;
|
||||
}
|
||||
});
|
||||
|
||||
async function seedFixture(opts?: {
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
StartupRefusalError,
|
||||
migrationRefusalError,
|
||||
shouldReportStartupFailure,
|
||||
} from "../startup-refusals.ts";
|
||||
|
||||
describe("migrationRefusalError", () => {
|
||||
const message = "PostgreSQL has pending migrations (…). Refusing to start.";
|
||||
|
||||
it("classifies a never-migrated database as a supervised-transient refusal", () => {
|
||||
const error = migrationRefusalError({ appliedMigrations: [], tableCount: 0 }, message);
|
||||
expect(error).toBeInstanceOf(StartupRefusalError);
|
||||
expect((error as StartupRefusalError).kind).toBe("schema-not-yet-migrated");
|
||||
expect(error.message).toContain("Refusing to start");
|
||||
});
|
||||
|
||||
it("keeps pending migrations on a migrated database as a plain, always-reported error", () => {
|
||||
const error = migrationRefusalError(
|
||||
{ appliedMigrations: ["0000_init.sql"], tableCount: 41 },
|
||||
message,
|
||||
);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect(error).not.toBeInstanceOf(StartupRefusalError);
|
||||
});
|
||||
|
||||
it("treats an empty journal beside existing tables as drift, not a fresh database", () => {
|
||||
// A wiped or never-populated migration journal next to real tables is
|
||||
// a persistent failure; it must keep reporting.
|
||||
const error = migrationRefusalError({ appliedMigrations: [], tableCount: 17 }, message);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect(error).not.toBeInstanceOf(StartupRefusalError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldReportStartupFailure", () => {
|
||||
const refusal = new StartupRefusalError(
|
||||
"database-contract-unmet",
|
||||
"authenticated public deployments require DATABASE_URL",
|
||||
);
|
||||
|
||||
it("always reports non-refusal startup failures, managed cloud or not", () => {
|
||||
expect(shouldReportStartupFailure(new Error("boom"), {})).toBe(true);
|
||||
expect(
|
||||
shouldReportStartupFailure(new Error("boom"), {
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.com",
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("reports supervised-transient refusals outside managed-cloud deployments", () => {
|
||||
expect(shouldReportStartupFailure(refusal, {})).toBe(true);
|
||||
});
|
||||
|
||||
it("suppresses supervised-transient refusals when a cloud supervisor owns the deployment", () => {
|
||||
expect(
|
||||
shouldReportStartupFailure(refusal, {
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.com",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("treats a blank cloud origin as unset", () => {
|
||||
expect(shouldReportStartupFailure(refusal, { PAPERCLIP_CLOUD_API_ORIGIN: " " })).toBe(true);
|
||||
});
|
||||
|
||||
it("reports non-Error throwables unconditionally", () => {
|
||||
expect(
|
||||
shouldReportStartupFailure("string failure", {
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.com",
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
} from "./services/company-import-transfers.js";
|
||||
import { companyTransferRunService } from "./services/company-transfer-runs.js";
|
||||
import { healthRoutes } from "./routes/health.js";
|
||||
import { cloudRuntimeIdentityMiddleware } from "./middleware/cloud-runtime-identity.js";
|
||||
import { cloudRoutes } from "./routes/cloud.js";
|
||||
import { companyRoutes } from "./routes/companies.js";
|
||||
import { companySkillRoutes } from "./routes/company-skills.js";
|
||||
@@ -364,6 +365,7 @@ export async function createApp(
|
||||
bindHost: opts.bindHost,
|
||||
}),
|
||||
);
|
||||
app.use(cloudRuntimeIdentityMiddleware(db));
|
||||
// Connection-intent tools carry their own short-lived, run-bound bearer and
|
||||
// must be reachable by remote adapters that intentionally do not receive an
|
||||
// agent API key. Every request revalidates the active heartbeat row.
|
||||
|
||||
+35
-4
@@ -35,6 +35,11 @@ import detectPort from "detect-port";
|
||||
import { createApp } from "./app.js";
|
||||
import { loadConfig } from "./config.js";
|
||||
import { logger } from "./middleware/logger.js";
|
||||
import {
|
||||
StartupRefusalError,
|
||||
migrationRefusalError,
|
||||
shouldReportStartupFailure,
|
||||
} from "./startup-refusals.js";
|
||||
import {
|
||||
getManagedInstanceConfig,
|
||||
type ManagedInstanceConfig,
|
||||
@@ -101,6 +106,7 @@ import {
|
||||
finalizeServerShutdown,
|
||||
loadWithoutCoordinatedShutdownSignalHooks,
|
||||
} from "./shutdown.js";
|
||||
import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js";
|
||||
import { systemdNotify } from "./services/systemd-notify.js";
|
||||
import { flushInFlightRunLogMirrors } from "./services/run-log-store.js";
|
||||
import {
|
||||
@@ -242,12 +248,20 @@ export async function startServer(): Promise<StartedServer> {
|
||||
|
||||
const apply = autoApply ? true : await promptApplyMigrations(state.pendingMigrations);
|
||||
if (!apply) {
|
||||
throw new Error(
|
||||
// A database with zero applied migrations and zero tables has
|
||||
// never been migrated: under a managed-cloud supervisor that is
|
||||
// the expected first-boot race (the harness migrates and
|
||||
// restarts), so the refusal carries the supervised-transient
|
||||
// class. Applied history — or pre-existing tables beside an empty
|
||||
// journal — means drift and keeps the plain, always-reported
|
||||
// Error.
|
||||
throw migrationRefusalError(
|
||||
state,
|
||||
`${label} has pending migrations (${formatPendingMigrationSummary(state.pendingMigrations)}). ` +
|
||||
"Refusing to start against a stale schema. Run pnpm db:migrate or set PAPERCLIP_MIGRATION_AUTO_APPLY=true.",
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
logger.info({ pendingMigrations: state.pendingMigrations }, `Applying ${state.pendingMigrations.length} pending migrations for ${label}`);
|
||||
await applyPendingMigrations(connectionString);
|
||||
return "applied (pending migrations)";
|
||||
@@ -267,7 +281,13 @@ export async function startServer(): Promise<StartedServer> {
|
||||
return;
|
||||
}
|
||||
if (!config.databaseUrl) {
|
||||
throw new Error(
|
||||
// Under a managed-cloud supervisor a missing DATABASE_URL on boot
|
||||
// is the config-application race (the container can start before
|
||||
// the staged variables land), not operator error — the supervisor
|
||||
// restarts once the config holds. A malformed value below is a
|
||||
// real misconfiguration and stays an always-reported Error.
|
||||
throw new StartupRefusalError(
|
||||
"database-contract-unmet",
|
||||
"authenticated public deployments require DATABASE_URL or config.database.connectionString; refusing embedded PostgreSQL fallback",
|
||||
);
|
||||
}
|
||||
@@ -562,6 +582,12 @@ export async function startServer(): Promise<StartedServer> {
|
||||
startupDbInfo = { mode: "embedded-postgres", dataDir, port };
|
||||
}
|
||||
|
||||
// A claimed warm-pool stack may restart while its provider environment still
|
||||
// names the pool host. Restore the signed, durable identity before Better
|
||||
// Auth, routes, or child-runtime configuration capture any public URL.
|
||||
const restoredCloudRuntimeIdentity = await initializeCloudRuntimeIdentity(db as any);
|
||||
if (restoredCloudRuntimeIdentity) config = loadConfig();
|
||||
|
||||
if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) {
|
||||
throw new Error(
|
||||
`local_trusted mode requires loopback host binding (received: ${config.host}). ` +
|
||||
@@ -1810,7 +1836,12 @@ function isMainModule(metaUrl: string): boolean {
|
||||
if (isMainModule(import.meta.url)) {
|
||||
void startServer().catch(async (err) => {
|
||||
logger.error({ err }, "Paperclip server failed to start");
|
||||
captureException(err);
|
||||
// Supervised-transient refusals in managed-cloud deployments are an
|
||||
// expected provisioning phase (see startup-refusals.ts) — they log
|
||||
// and exit nonzero but do not page Sentry.
|
||||
if (shouldReportStartupFailure(err)) {
|
||||
captureException(err);
|
||||
}
|
||||
await shutdownSentry();
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { RequestHandler } from "express";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { logger } from "./logger.js";
|
||||
import {
|
||||
applyCloudRuntimeIdentityAssertion,
|
||||
CLOUD_RUNTIME_IDENTITY_HEADER,
|
||||
} from "../services/cloud-runtime-identity.js";
|
||||
|
||||
/**
|
||||
* Accepts Cloud's signed identity only on the existing bootstrap health call.
|
||||
* The JWS is sufficient authorization; the browser-facing proxy strips this
|
||||
* header, and possession of the shared tenant-session token cannot mint it.
|
||||
*/
|
||||
export function cloudRuntimeIdentityMiddleware(db: Db): RequestHandler {
|
||||
return async (req, res, next) => {
|
||||
const assertion = req.get(CLOUD_RUNTIME_IDENTITY_HEADER)?.trim();
|
||||
if (!assertion) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
if (req.method !== "GET" || req.path !== "/api/health") {
|
||||
res.status(400).json({ error: "cloud_runtime_identity_wrong_endpoint" });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await applyCloudRuntimeIdentityAssertion({ db, compactJws: assertion });
|
||||
next();
|
||||
} catch (error) {
|
||||
logger.warn({ err: error }, "Rejected Cloud runtime identity assertion");
|
||||
res.status(401).json({ error: "invalid_cloud_runtime_identity" });
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -57,6 +57,7 @@ import {
|
||||
tooManyRequests
|
||||
} from "../errors.js";
|
||||
import { getHiddenSettings } from "../services/settings-visibility.js";
|
||||
import { runtimeCanonicalOrigin } from "../services/cloud-runtime-identity.js";
|
||||
|
||||
/**
|
||||
* Floor: when the hosting operator hides the Instance Access surface
|
||||
@@ -153,6 +154,8 @@ function requestBaseUrl(req: Request) {
|
||||
}
|
||||
|
||||
function resolveBaseUrl(req: Request, authPublicBaseUrl?: string): string {
|
||||
const runtimeOrigin = runtimeCanonicalOrigin();
|
||||
if (runtimeOrigin) return runtimeOrigin;
|
||||
if (authPublicBaseUrl) return authPublicBaseUrl.replace(/\/+$/, "");
|
||||
return requestBaseUrl(req);
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
isCloudManagedInstance,
|
||||
type CloudInstanceEnv,
|
||||
} from "../services/cloud-instance.js";
|
||||
import { getCloudRuntimeIdentity } from "../services/cloud-runtime-identity.js";
|
||||
import { getHiddenSettings } from "../services/settings-visibility.js";
|
||||
import {
|
||||
inspectDatabaseBackupHealth,
|
||||
@@ -88,12 +89,19 @@ function redactedDatabaseBackupHealth(databaseBackup: DatabaseBackupHealthStatus
|
||||
function getCloudHealthStatus(env: CloudInstanceEnv) {
|
||||
const context = getCloudStackContext(env);
|
||||
if (!context) return undefined;
|
||||
const runtimeIdentity = env === process.env ? getCloudRuntimeIdentity() : null;
|
||||
|
||||
return {
|
||||
managed: true as const,
|
||||
managedBy: "paperclip-cloud" as const,
|
||||
stackSlug: context.stackSlug,
|
||||
cloudBaseUrl: context.cloudOrigin,
|
||||
...(runtimeIdentity ? {
|
||||
runtimeIdentity: {
|
||||
canonicalOrigin: runtimeIdentity.canonicalOrigin,
|
||||
stackSlug: runtimeIdentity.stackSlug,
|
||||
},
|
||||
} : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ import {
|
||||
type PaperclipCloudConnector,
|
||||
paperclipCloudConnectorCapabilitiesFromEnv,
|
||||
} from "../services/paperclip-cloud-connector.js";
|
||||
import { runtimeCanonicalOrigin } from "../services/cloud-runtime-identity.js";
|
||||
import {
|
||||
completePaperclipCloudConnectorEnrollment,
|
||||
loadPaperclipCloudConnectorIdentity,
|
||||
@@ -298,12 +299,14 @@ export function toolAccessRoutes(
|
||||
}
|
||||
|
||||
function configuredPublicBaseUrl() {
|
||||
const runtimeOrigin = runtimeCanonicalOrigin();
|
||||
if (runtimeOrigin) return runtimeOrigin;
|
||||
const raw = (
|
||||
process.env.PAPERCLIP_PUBLIC_URL?.trim()
|
||||
|| process.env.PAPERCLIP_AUTH_PUBLIC_BASE_URL?.trim()
|
||||
process.env.PAPERCLIP_AUTH_PUBLIC_BASE_URL?.trim()
|
||||
|| process.env.BETTER_AUTH_URL?.trim()
|
||||
|| process.env.BETTER_AUTH_BASE_URL?.trim()
|
||||
|| options.authPublicBaseUrl?.trim()
|
||||
|| process.env.PAPERCLIP_PUBLIC_URL?.trim()
|
||||
|| process.env.PAPERCLIP_MANAGED_RUNTIME_PUBLIC_URL?.trim()
|
||||
);
|
||||
if (!raw) return null;
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import { generateKeyPairSync, sign, type KeyObject } from "node:crypto";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
CLOUD_RUNTIME_IDENTITY_AUDIENCE,
|
||||
CLOUD_RUNTIME_IDENTITY_ISSUER,
|
||||
CLOUD_RUNTIME_IDENTITY_JWS_TYPE,
|
||||
runtimeCanonicalOrigin,
|
||||
runtimePublicOrigin,
|
||||
verifyCloudRuntimeIdentityAssertion,
|
||||
} from "./cloud-runtime-identity.js";
|
||||
|
||||
const STACK_ID = "stack-pool-123";
|
||||
const POOL_ORIGIN = "https://pool-123.staging.paperclip.app";
|
||||
const CANONICAL_ORIGIN = "https://gonzo.staging.paperclip.app";
|
||||
const NOW = new Date("2099-01-01T00:00:00.000Z");
|
||||
const pair = generateKeyPairSync("ed25519");
|
||||
const publicJwk = {
|
||||
...pair.publicKey.export({ format: "jwk" }),
|
||||
kid: "runtime-identity-test-key",
|
||||
use: "sig",
|
||||
alg: "EdDSA",
|
||||
};
|
||||
const env = {
|
||||
PAPERCLIP_CLOUD_STACK_ID: STACK_ID,
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [publicJwk] }),
|
||||
} as NodeJS.ProcessEnv;
|
||||
|
||||
function encodeJson(value: Record<string, unknown>) {
|
||||
return Buffer.from(JSON.stringify(value)).toString("base64url");
|
||||
}
|
||||
|
||||
function assertion(input: {
|
||||
claims?: Record<string, unknown>;
|
||||
header?: Record<string, unknown>;
|
||||
signingKey?: KeyObject;
|
||||
} = {}) {
|
||||
const iat = Math.floor(NOW.getTime() / 1000);
|
||||
const header = encodeJson({
|
||||
alg: "EdDSA",
|
||||
typ: CLOUD_RUNTIME_IDENTITY_JWS_TYPE,
|
||||
kid: publicJwk.kid,
|
||||
...input.header,
|
||||
});
|
||||
const payload = encodeJson({
|
||||
v: 1,
|
||||
iss: CLOUD_RUNTIME_IDENTITY_ISSUER,
|
||||
aud: CLOUD_RUNTIME_IDENTITY_AUDIENCE,
|
||||
sub: STACK_ID,
|
||||
claimId: "pool-entry-123",
|
||||
previousOrigin: POOL_ORIGIN,
|
||||
canonicalOrigin: CANONICAL_ORIGIN,
|
||||
stackSlug: "gonzo",
|
||||
iat,
|
||||
exp: iat + 300,
|
||||
...input.claims,
|
||||
});
|
||||
const signature = sign(
|
||||
null,
|
||||
Buffer.from(`${header}.${payload}`, "ascii"),
|
||||
input.signingKey ?? pair.privateKey,
|
||||
).toString("base64url");
|
||||
return `${header}.${payload}.${signature}`;
|
||||
}
|
||||
|
||||
function verifyAssertion(compactJws: string, overrides: Partial<NodeJS.ProcessEnv> = {}) {
|
||||
return verifyCloudRuntimeIdentityAssertion({
|
||||
compactJws,
|
||||
env: { ...env, ...overrides },
|
||||
now: NOW,
|
||||
expectedPreviousOrigin: POOL_ORIGIN,
|
||||
});
|
||||
}
|
||||
|
||||
describe("verifyCloudRuntimeIdentityAssertion", () => {
|
||||
it("preserves distinct self-hosted public and authentication origins", () => {
|
||||
const selfHostedEnv = {
|
||||
PAPERCLIP_PUBLIC_URL: "https://app.example.test",
|
||||
PAPERCLIP_AUTH_PUBLIC_BASE_URL: "https://auth.example.test",
|
||||
PAPERCLIP_API_URL: "https://api.example.test",
|
||||
} as NodeJS.ProcessEnv;
|
||||
|
||||
expect(runtimePublicOrigin(selfHostedEnv)).toBe("https://app.example.test");
|
||||
expect(runtimeCanonicalOrigin()).toBeNull();
|
||||
});
|
||||
|
||||
it("accepts a Cloud-signed claim for this exact stack and pool origin", () => {
|
||||
expect(verifyAssertion(assertion())).toMatchObject({
|
||||
sub: STACK_ID,
|
||||
claimId: "pool-entry-123",
|
||||
canonicalOrigin: CANONICAL_ORIGIN,
|
||||
stackSlug: "gonzo",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects forged signatures and unknown signing keys", () => {
|
||||
const attacker = generateKeyPairSync("ed25519");
|
||||
expect(() => verifyAssertion(assertion({ signingKey: attacker.privateKey }))).toThrow("signature is invalid");
|
||||
expect(() => verifyAssertion(assertion({ header: { kid: "unknown" } }))).toThrow("unknown signing key");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["expired", { exp: Math.floor(NOW.getTime() / 1000) - 1 }],
|
||||
["wrong audience", { aud: "someone-else" }],
|
||||
["cross-stack", { sub: "stack-someone-else" }],
|
||||
["wrong pool origin", { previousOrigin: "https://someone-else.staging.paperclip.app" }],
|
||||
["non-HTTPS destination", { canonicalOrigin: "http://gonzo.staging.paperclip.app" }],
|
||||
["path-bearing destination", { canonicalOrigin: `${CANONICAL_ORIGIN}/GON` }],
|
||||
["slug mismatch", { stackSlug: "kermit" }],
|
||||
["partial claims", { claimId: undefined }],
|
||||
])("rejects %s assertions", (_label, claims) => {
|
||||
expect(() => verifyAssertion(assertion({ claims }))).toThrow();
|
||||
});
|
||||
|
||||
it("rejects an altered signed destination", () => {
|
||||
const valid = assertion();
|
||||
const [header, payload, signature] = valid.split(".");
|
||||
const altered = encodeJson({
|
||||
...JSON.parse(Buffer.from(payload!, "base64url").toString("utf8")),
|
||||
canonicalOrigin: "https://attacker.example.com",
|
||||
});
|
||||
expect(() => verifyAssertion(`${header}.${altered}.${signature}`)).toThrow("signature is invalid");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,429 @@
|
||||
import { createPublicKey, timingSafeEqual, verify, type JsonWebKey } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { instanceSettings } from "@paperclipai/db";
|
||||
|
||||
export const CLOUD_RUNTIME_IDENTITY_HEADER = "x-paperclip-cloud-runtime-identity";
|
||||
export const CLOUD_RUNTIME_IDENTITY_AUDIENCE = "paperclip-runtime-identity/v1";
|
||||
export const CLOUD_RUNTIME_IDENTITY_ISSUER = "paperclip-cloud";
|
||||
export const CLOUD_RUNTIME_IDENTITY_JWS_TYPE = "paperclip-cloud-runtime-identity+jwt";
|
||||
|
||||
const SINGLETON_KEY = "cloud-runtime-identity/v1";
|
||||
const MAX_ASSERTION_LIFETIME_SECONDS = 10 * 60;
|
||||
const MAX_CLOCK_SKEW_SECONDS = 30;
|
||||
const STACK_SLUG_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
||||
|
||||
type PersistedRuntimeIdentity = {
|
||||
stackId: string;
|
||||
claimId: string;
|
||||
previousOrigin: string;
|
||||
canonicalOrigin: string;
|
||||
stackSlug: string;
|
||||
appliedAt: Date;
|
||||
};
|
||||
|
||||
type RuntimeIdentityDb = Pick<Db, "select" | "insert">;
|
||||
|
||||
export type CloudRuntimeIdentitySnapshot = {
|
||||
stackId: string;
|
||||
claimId: string;
|
||||
previousOrigin: string;
|
||||
canonicalOrigin: string;
|
||||
stackSlug: string;
|
||||
appliedAt: Date;
|
||||
};
|
||||
|
||||
export type RuntimeIdentityClaims = {
|
||||
v: 1;
|
||||
iss: typeof CLOUD_RUNTIME_IDENTITY_ISSUER;
|
||||
aud: typeof CLOUD_RUNTIME_IDENTITY_AUDIENCE;
|
||||
sub: string;
|
||||
claimId: string;
|
||||
previousOrigin: string;
|
||||
canonicalOrigin: string;
|
||||
stackSlug: string;
|
||||
iat: number;
|
||||
exp: number;
|
||||
};
|
||||
|
||||
let initialized = false;
|
||||
let startupOrigin: string | null = null;
|
||||
let currentIdentity: CloudRuntimeIdentitySnapshot | null = null;
|
||||
|
||||
function nonEmpty(value: string | undefined): string | null {
|
||||
const normalized = value?.trim();
|
||||
return normalized ? normalized : null;
|
||||
}
|
||||
|
||||
function exactHttpsOrigin(value: unknown): string | null {
|
||||
if (typeof value !== "string" || value.length === 0 || value.length > 2048) return null;
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
if (
|
||||
parsed.protocol !== "https:"
|
||||
|| parsed.username
|
||||
|| parsed.password
|
||||
|| parsed.pathname !== "/"
|
||||
|| parsed.search
|
||||
|| parsed.hash
|
||||
|| parsed.origin !== value
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return parsed.origin;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function configuredStartupOrigin(env: NodeJS.ProcessEnv): string | null {
|
||||
const candidate = nonEmpty(env.PAPERCLIP_PUBLIC_URL)
|
||||
?? nonEmpty(env.PAPERCLIP_AUTH_PUBLIC_BASE_URL)
|
||||
?? nonEmpty(env.PAPERCLIP_API_URL);
|
||||
return candidate ? exactHttpsOrigin(candidate) : null;
|
||||
}
|
||||
|
||||
function snapshot(row: PersistedRuntimeIdentity): CloudRuntimeIdentitySnapshot {
|
||||
return {
|
||||
stackId: row.stackId,
|
||||
claimId: row.claimId,
|
||||
previousOrigin: row.previousOrigin,
|
||||
canonicalOrigin: row.canonicalOrigin,
|
||||
stackSlug: row.stackSlug,
|
||||
appliedAt: row.appliedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function parsePersistedIdentity(row: {
|
||||
general: Record<string, unknown>;
|
||||
createdAt: Date;
|
||||
}): PersistedRuntimeIdentity {
|
||||
const value = row.general;
|
||||
if (
|
||||
value.v !== 1
|
||||
|| typeof value.stackId !== "string"
|
||||
|| typeof value.claimId !== "string"
|
||||
|| typeof value.previousOrigin !== "string"
|
||||
|| typeof value.canonicalOrigin !== "string"
|
||||
|| typeof value.stackSlug !== "string"
|
||||
) {
|
||||
throw new Error("Persisted Cloud runtime identity is malformed");
|
||||
}
|
||||
return {
|
||||
stackId: value.stackId,
|
||||
claimId: value.claimId,
|
||||
previousOrigin: value.previousOrigin,
|
||||
canonicalOrigin: value.canonicalOrigin,
|
||||
stackSlug: value.stackSlug,
|
||||
appliedAt: row.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
async function readPersistedIdentity(db: RuntimeIdentityDb): Promise<PersistedRuntimeIdentity | null> {
|
||||
const row = await db
|
||||
.select({
|
||||
general: instanceSettings.general,
|
||||
createdAt: instanceSettings.createdAt,
|
||||
})
|
||||
.from(instanceSettings)
|
||||
.where(eq(instanceSettings.singletonKey, SINGLETON_KEY))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row ? parsePersistedIdentity(row) : null;
|
||||
}
|
||||
|
||||
function applyCompatibilityEnvironment(identity: CloudRuntimeIdentitySnapshot, env: NodeJS.ProcessEnv) {
|
||||
const hostname = new URL(identity.canonicalOrigin).hostname;
|
||||
env.PAPERCLIP_PUBLIC_URL = identity.canonicalOrigin;
|
||||
env.PAPERCLIP_AUTH_PUBLIC_BASE_URL = identity.canonicalOrigin;
|
||||
env.PAPERCLIP_API_URL = identity.canonicalOrigin;
|
||||
env.PAPERCLIP_PRIMARY_HOST = hostname;
|
||||
env.PAPERCLIP_STACK_SLUG = identity.stackSlug;
|
||||
|
||||
const existingCandidates = (() => {
|
||||
try {
|
||||
const parsed = JSON.parse(env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON ?? "[]");
|
||||
return Array.isArray(parsed) ? parsed.filter((value): value is string => typeof value === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})();
|
||||
env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = JSON.stringify([
|
||||
identity.canonicalOrigin,
|
||||
...existingCandidates.filter((candidate) => candidate !== identity.canonicalOrigin),
|
||||
]);
|
||||
}
|
||||
|
||||
function assertPersistedIdentityMatchesStack(row: PersistedRuntimeIdentity, env: NodeJS.ProcessEnv) {
|
||||
const configuredStackId = nonEmpty(env.PAPERCLIP_CLOUD_STACK_ID);
|
||||
if (!configuredStackId || configuredStackId !== row.stackId) {
|
||||
throw new Error("Persisted Cloud runtime identity does not match PAPERCLIP_CLOUD_STACK_ID");
|
||||
}
|
||||
if (!exactHttpsOrigin(row.previousOrigin) || !exactHttpsOrigin(row.canonicalOrigin)) {
|
||||
throw new Error("Persisted Cloud runtime identity contains an invalid origin");
|
||||
}
|
||||
if (!STACK_SLUG_PATTERN.test(row.stackSlug) || new URL(row.canonicalOrigin).hostname.split(".")[0] !== row.stackSlug) {
|
||||
throw new Error("Persisted Cloud runtime identity contains an invalid stack slug");
|
||||
}
|
||||
}
|
||||
|
||||
/** Load the durable claim before auth, routes, and child-runtime configuration. */
|
||||
export async function initializeCloudRuntimeIdentity(
|
||||
db: Db,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<CloudRuntimeIdentitySnapshot | null> {
|
||||
startupOrigin = configuredStartupOrigin(env);
|
||||
// Self-hosted servers have no Cloud stack identity to restore. Avoid touching
|
||||
// the singleton table on that path; besides keeping the feature inert, this
|
||||
// preserves lightweight startup/test database seams that intentionally do
|
||||
// not construct a database client.
|
||||
if (!nonEmpty(env.PAPERCLIP_CLOUD_STACK_ID)) {
|
||||
initialized = true;
|
||||
currentIdentity = null;
|
||||
return null;
|
||||
}
|
||||
const row = await readPersistedIdentity(db);
|
||||
initialized = true;
|
||||
if (!row) {
|
||||
currentIdentity = null;
|
||||
return null;
|
||||
}
|
||||
assertPersistedIdentityMatchesStack(row, env);
|
||||
currentIdentity = snapshot(row);
|
||||
applyCompatibilityEnvironment(currentIdentity, env);
|
||||
return currentIdentity;
|
||||
}
|
||||
|
||||
export function getCloudRuntimeIdentity(): CloudRuntimeIdentitySnapshot | null {
|
||||
return currentIdentity ? { ...currentIdentity } : null;
|
||||
}
|
||||
|
||||
/** The live canonical origin, falling back to startup configuration off Cloud. */
|
||||
export function runtimePublicOrigin(env: NodeJS.ProcessEnv = process.env): string | null {
|
||||
if (env === process.env && currentIdentity) return currentIdentity.canonicalOrigin;
|
||||
const candidate = nonEmpty(env.PAPERCLIP_PUBLIC_URL)
|
||||
?? nonEmpty(env.PAPERCLIP_AUTH_PUBLIC_BASE_URL)
|
||||
?? nonEmpty(env.PAPERCLIP_API_URL);
|
||||
if (!candidate) return null;
|
||||
try {
|
||||
return new URL(candidate).origin;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The asserted Cloud origin only. Callers retain their non-Cloud precedence. */
|
||||
export function runtimeCanonicalOrigin(): string | null {
|
||||
return currentIdentity?.canonicalOrigin ?? null;
|
||||
}
|
||||
|
||||
function decodeJsonPart(part: string, label: string): Record<string, unknown> {
|
||||
if (!/^[A-Za-z0-9_-]+$/.test(part)) throw new Error(`Cloud runtime identity has an invalid ${label}`);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(Buffer.from(part, "base64url").toString("utf8"));
|
||||
} catch {
|
||||
throw new Error(`Cloud runtime identity has an invalid ${label}`);
|
||||
}
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
throw new Error(`Cloud runtime identity has an invalid ${label}`);
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function publicKeyForKid(env: NodeJS.ProcessEnv, kid: string) {
|
||||
const raw = nonEmpty(env.PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS);
|
||||
if (!raw) throw new Error("PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS is not configured");
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new Error("PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS is invalid");
|
||||
}
|
||||
const keys = parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as { keys?: unknown }).keys
|
||||
: undefined;
|
||||
if (!Array.isArray(keys)) throw new Error("PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS is invalid");
|
||||
const matches = keys.filter((candidate): candidate is JsonWebKey & { kid: string } => {
|
||||
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) return false;
|
||||
const key = candidate as JsonWebKey & { kid?: unknown };
|
||||
return key.kid === kid;
|
||||
});
|
||||
if (matches.length !== 1) throw new Error("Cloud runtime identity uses an unknown signing key");
|
||||
const jwk = matches[0];
|
||||
if (jwk.kty !== "OKP" || jwk.crv !== "Ed25519" || jwk.use !== "sig" || jwk.alg !== "EdDSA" || !jwk.x || jwk.d) {
|
||||
throw new Error("Cloud runtime identity signing key is invalid");
|
||||
}
|
||||
return createPublicKey({ key: jwk, format: "jwk" });
|
||||
}
|
||||
|
||||
function verifyClaims(input: {
|
||||
compactJws: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
now: Date;
|
||||
}): RuntimeIdentityClaims {
|
||||
const parts = input.compactJws.split(".");
|
||||
if (parts.length !== 3 || parts.some((part) => part.length === 0)) {
|
||||
throw new Error("Cloud runtime identity assertion is not a compact JWS");
|
||||
}
|
||||
const [encodedHeader, encodedPayload, encodedSignature] = parts;
|
||||
const header = decodeJsonPart(encodedHeader, "protected header");
|
||||
if (
|
||||
header.alg !== "EdDSA"
|
||||
|| header.typ !== CLOUD_RUNTIME_IDENTITY_JWS_TYPE
|
||||
|| typeof header.kid !== "string"
|
||||
|| !header.kid
|
||||
) {
|
||||
throw new Error("Cloud runtime identity protected header is invalid");
|
||||
}
|
||||
const key = publicKeyForKid(input.env, header.kid);
|
||||
const signature = Buffer.from(encodedSignature, "base64url");
|
||||
const signingInput = Buffer.from(`${encodedHeader}.${encodedPayload}`, "ascii");
|
||||
if (!verify(null, signingInput, key, signature)) {
|
||||
throw new Error("Cloud runtime identity signature is invalid");
|
||||
}
|
||||
|
||||
const payload = decodeJsonPart(encodedPayload, "payload");
|
||||
const nowSeconds = Math.floor(input.now.getTime() / 1000);
|
||||
if (
|
||||
payload.v !== 1
|
||||
|| payload.iss !== CLOUD_RUNTIME_IDENTITY_ISSUER
|
||||
|| payload.aud !== CLOUD_RUNTIME_IDENTITY_AUDIENCE
|
||||
|| typeof payload.sub !== "string"
|
||||
|| typeof payload.claimId !== "string"
|
||||
|| typeof payload.previousOrigin !== "string"
|
||||
|| typeof payload.canonicalOrigin !== "string"
|
||||
|| typeof payload.stackSlug !== "string"
|
||||
|| typeof payload.iat !== "number"
|
||||
|| !Number.isInteger(payload.iat)
|
||||
|| typeof payload.exp !== "number"
|
||||
|| !Number.isInteger(payload.exp)
|
||||
) {
|
||||
throw new Error("Cloud runtime identity claims are incomplete");
|
||||
}
|
||||
if (
|
||||
payload.exp <= nowSeconds
|
||||
|| payload.iat > nowSeconds + MAX_CLOCK_SKEW_SECONDS
|
||||
|| payload.exp <= payload.iat
|
||||
|| payload.exp - payload.iat > MAX_ASSERTION_LIFETIME_SECONDS
|
||||
) {
|
||||
throw new Error("Cloud runtime identity assertion is expired or has an invalid lifetime");
|
||||
}
|
||||
return payload as RuntimeIdentityClaims;
|
||||
}
|
||||
|
||||
/** Verify that an assertion is signed for this exact, still-unclaimed instance. */
|
||||
export function verifyCloudRuntimeIdentityAssertion(input: {
|
||||
compactJws: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
now?: Date;
|
||||
expectedPreviousOrigin: string | null;
|
||||
}): RuntimeIdentityClaims {
|
||||
const env = input.env ?? process.env;
|
||||
const claims = verifyClaims({ compactJws: input.compactJws, env, now: input.now ?? new Date() });
|
||||
const configuredStackId = nonEmpty(env.PAPERCLIP_CLOUD_STACK_ID);
|
||||
if (!configuredStackId || claims.sub !== configuredStackId) {
|
||||
throw new Error("Cloud runtime identity stack does not match this instance");
|
||||
}
|
||||
const previousOrigin = exactHttpsOrigin(claims.previousOrigin);
|
||||
const canonicalOrigin = exactHttpsOrigin(claims.canonicalOrigin);
|
||||
if (!previousOrigin || !canonicalOrigin || previousOrigin !== input.expectedPreviousOrigin) {
|
||||
throw new Error("Cloud runtime identity previous or canonical origin is invalid");
|
||||
}
|
||||
if (
|
||||
!STACK_SLUG_PATTERN.test(claims.stackSlug)
|
||||
|| new URL(canonicalOrigin).hostname.split(".")[0] !== claims.stackSlug
|
||||
|| claims.claimId.length > 256
|
||||
|| claims.claimId.trim() !== claims.claimId
|
||||
|| !claims.claimId
|
||||
) {
|
||||
throw new Error("Cloud runtime identity destination is invalid");
|
||||
}
|
||||
return claims;
|
||||
}
|
||||
|
||||
function assertionsEqual(row: PersistedRuntimeIdentity, claims: RuntimeIdentityClaims): boolean {
|
||||
const left = Buffer.from(JSON.stringify([
|
||||
row.stackId,
|
||||
row.claimId,
|
||||
row.previousOrigin,
|
||||
row.canonicalOrigin,
|
||||
row.stackSlug,
|
||||
]));
|
||||
const right = Buffer.from(JSON.stringify([
|
||||
claims.sub,
|
||||
claims.claimId,
|
||||
claims.previousOrigin,
|
||||
claims.canonicalOrigin,
|
||||
claims.stackSlug,
|
||||
]));
|
||||
return left.length === right.length && timingSafeEqual(left, right);
|
||||
}
|
||||
|
||||
/** Verify and durably apply the one-time Cloud claim assertion. */
|
||||
export async function applyCloudRuntimeIdentityAssertion(input: {
|
||||
db: Db;
|
||||
compactJws: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
now?: Date;
|
||||
}): Promise<CloudRuntimeIdentitySnapshot> {
|
||||
const env = input.env ?? process.env;
|
||||
if (!initialized) throw new Error("Cloud runtime identity provider is not initialized");
|
||||
const claims = verifyCloudRuntimeIdentityAssertion({
|
||||
compactJws: input.compactJws,
|
||||
env,
|
||||
now: input.now,
|
||||
// After a natural restart the provider env may already be canonical, but
|
||||
// an identical retry of the original claim is still safe and idempotent.
|
||||
// The durable row preserves the pool origin that assertion had to match
|
||||
// on first application.
|
||||
expectedPreviousOrigin: currentIdentity?.previousOrigin ?? startupOrigin,
|
||||
});
|
||||
const previousOrigin = claims.previousOrigin;
|
||||
const canonicalOrigin = claims.canonicalOrigin;
|
||||
|
||||
const row = await input.db.transaction(async (tx) => {
|
||||
const existing = await readPersistedIdentity(tx);
|
||||
if (existing) {
|
||||
if (!assertionsEqual(existing, claims)) {
|
||||
throw new Error("Cloud runtime identity is already claimed by another assertion");
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
const now = input.now ?? new Date();
|
||||
await tx
|
||||
.insert(instanceSettings)
|
||||
.values({
|
||||
singletonKey: SINGLETON_KEY,
|
||||
general: {
|
||||
v: 1,
|
||||
stackId: claims.sub,
|
||||
claimId: claims.claimId,
|
||||
previousOrigin,
|
||||
canonicalOrigin,
|
||||
stackSlug: claims.stackSlug,
|
||||
},
|
||||
experimental: {},
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.onConflictDoNothing({ target: instanceSettings.singletonKey });
|
||||
const durable = await readPersistedIdentity(tx);
|
||||
if (!durable || !assertionsEqual(durable, claims)) {
|
||||
throw new Error("Cloud runtime identity is already claimed by another assertion");
|
||||
}
|
||||
return durable;
|
||||
});
|
||||
|
||||
currentIdentity = snapshot(row);
|
||||
applyCompatibilityEnvironment(currentIdentity, env);
|
||||
return { ...currentIdentity };
|
||||
}
|
||||
|
||||
/** Test seam for modules that intentionally share a process. */
|
||||
export function resetCloudRuntimeIdentityForTests() {
|
||||
initialized = false;
|
||||
startupOrigin = null;
|
||||
currentIdentity = null;
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSy
|
||||
import path from "node:path";
|
||||
|
||||
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
|
||||
import { runtimePublicOrigin } from "./cloud-runtime-identity.js";
|
||||
|
||||
const IDENTITY_VERSION = 1;
|
||||
const ENROLLMENT_FILE = "paperclip-cloud-connector.json";
|
||||
@@ -90,7 +91,8 @@ export function paperclipCloudConnectorEnrollmentStatus(
|
||||
origins: [],
|
||||
};
|
||||
}
|
||||
const publicOrigin = env.PAPERCLIP_PUBLIC_URL ? normalizeInstanceOrigin(env.PAPERCLIP_PUBLIC_URL) : undefined;
|
||||
const resolvedOrigin = runtimePublicOrigin(env);
|
||||
const publicOrigin = resolvedOrigin ? normalizeInstanceOrigin(resolvedOrigin) : undefined;
|
||||
return {
|
||||
configured: true,
|
||||
status: "active",
|
||||
|
||||
@@ -77,6 +77,7 @@ import {
|
||||
import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./issue-assignment-wakeup.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
||||
import { runtimePublicOrigin } from "./cloud-runtime-identity.js";
|
||||
|
||||
const OPEN_ISSUE_STATUSES = ["backlog", "todo", "in_progress", "in_review", "blocked"];
|
||||
const LIVE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
|
||||
@@ -102,6 +103,12 @@ const WEEKDAY_INDEX: Record<string, number> = {
|
||||
Sat: 6,
|
||||
};
|
||||
|
||||
export function routineWebhookUrl(publicId: string): string {
|
||||
const baseUrl = runtimePublicOrigin() ?? process.env.PAPERCLIP_API_URL?.trim();
|
||||
if (!baseUrl) throw new Error("PAPERCLIP_API_URL is required to create a routine webhook");
|
||||
return `${baseUrl.replace(/\/+$/, "")}/api/routine-triggers/public/${publicId}/fire`;
|
||||
}
|
||||
|
||||
type ExecutionIssueTransientFailureStatus = (typeof EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES)[number];
|
||||
|
||||
function executionIssueTransientFailureReason(status: ExecutionIssueTransientFailureStatus) {
|
||||
@@ -2438,7 +2445,7 @@ export function routineService(
|
||||
const created = await createWebhookSecret(routine.companyId, routine.id, actor);
|
||||
secretId = created.secret.id;
|
||||
secretMaterial = {
|
||||
webhookUrl: `${process.env.PAPERCLIP_API_URL}/api/routine-triggers/public/${publicId}/fire`,
|
||||
webhookUrl: routineWebhookUrl(publicId),
|
||||
webhookSecret: created.secretValue,
|
||||
};
|
||||
}
|
||||
@@ -2621,7 +2628,7 @@ export function routineService(
|
||||
return {
|
||||
trigger: trigger as RoutineTrigger,
|
||||
secretMaterial: {
|
||||
webhookUrl: `${process.env.PAPERCLIP_API_URL}/api/routine-triggers/public/${existing.publicId}/fire`,
|
||||
webhookUrl: routineWebhookUrl(existing.publicId),
|
||||
webhookSecret: secretValue,
|
||||
},
|
||||
revision,
|
||||
@@ -2701,7 +2708,7 @@ export function routineService(
|
||||
secretId: created.secret.id,
|
||||
secretMaterial: {
|
||||
triggerId: trigger.id,
|
||||
webhookUrl: `${process.env.PAPERCLIP_API_URL}/api/routine-triggers/public/${publicId}/fire`,
|
||||
webhookUrl: routineWebhookUrl(publicId),
|
||||
webhookSecret: created.secretValue,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* Deliberate boot refusals and whether they should page Sentry.
|
||||
*
|
||||
* Some startup preconditions the server must not repair itself: an
|
||||
* unmigrated schema when auto-apply is off (the operator's migration
|
||||
* runner owns schema), or an unmet database contract for authenticated
|
||||
* public deployments. The server logs the refusal and exits nonzero so
|
||||
* whatever supervises the deployment can act.
|
||||
*
|
||||
* In supervised managed-cloud deployments (`PAPERCLIP_CLOUD_API_ORIGIN`
|
||||
* set), two of these refusals are a routine provisioning phase rather
|
||||
* than an incident: a freshly created stack's app container boots
|
||||
* before the harness has migrated the empty database or finished
|
||||
* applying its committed configuration, crash-loops briefly, and is
|
||||
* restarted by the harness once the precondition holds. Reporting every
|
||||
* such boot to Sentry buries real errors under hundreds of expected
|
||||
* events per fleet build batch, so the crash handler skips the capture
|
||||
* for exactly this class — the refusal still logs and still exits
|
||||
* nonzero. Everywhere else (self-hosted, local dev) reporting is
|
||||
* unchanged.
|
||||
*/
|
||||
|
||||
export type StartupRefusalKind =
|
||||
| "schema-not-yet-migrated"
|
||||
| "database-contract-unmet";
|
||||
|
||||
/**
|
||||
* A boot refusal whose remedy belongs to the deployment's supervisor.
|
||||
* Only refusals that are *expected transients* under managed-cloud
|
||||
* provisioning use this class; refusals that always indicate operator
|
||||
* error (schema drift, a malformed DATABASE_URL) stay plain `Error`s.
|
||||
*/
|
||||
export class StartupRefusalError extends Error {
|
||||
readonly kind: StartupRefusalKind;
|
||||
|
||||
constructor(kind: StartupRefusalKind, message: string) {
|
||||
super(message);
|
||||
this.name = "StartupRefusalError";
|
||||
this.kind = kind;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Chooses the error class for a pending-migrations refusal. A database
|
||||
* with zero applied migrations AND zero tables is not stale — it has
|
||||
* never been migrated at all, which under a supervisor means "not yet"
|
||||
* rather than "drifted". Any applied history, or any pre-existing
|
||||
* tables beside an empty or wiped migration journal, makes pending
|
||||
* migrations a drift signal that must keep reporting.
|
||||
*/
|
||||
export function migrationRefusalError(
|
||||
state: { appliedMigrations: string[]; tableCount: number },
|
||||
message: string,
|
||||
): Error {
|
||||
const neverMigrated = state.appliedMigrations.length === 0 && state.tableCount === 0;
|
||||
return neverMigrated
|
||||
? new StartupRefusalError("schema-not-yet-migrated", message)
|
||||
: new Error(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a startup failure should be captured to Sentry. Everything
|
||||
* reports except a supervised-transient refusal in a managed-cloud
|
||||
* deployment.
|
||||
*/
|
||||
export function shouldReportStartupFailure(
|
||||
error: unknown,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): boolean {
|
||||
if (!(error instanceof StartupRefusalError)) return true;
|
||||
const cloudOrigin = env.PAPERCLIP_CLOUD_API_ORIGIN?.trim();
|
||||
return !cloudOrigin;
|
||||
}
|
||||
Reference in new issue
Block a user