mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 07:55:47 +01:00
feat(server): bind idle credential eligibility to incoming wake proof (#18639)
Reviewed and verified at d1e3afa4af.
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
0b87308d5d
commit
5877d751c0
31 files changed
+2071
-53
No files matched your search
@@ -136,6 +136,7 @@ import { notifyChatDeliveryWork, notifyChatPublicationWork } from "./chat-work-n
|
||||
|
||||
import { voiceSessionStore } from "./voice/voice-session-store.js";
|
||||
import { createChatRuntime, type ChatRuntime } from "./chat-runtime.js";
|
||||
import { registerIdleLocalWorkProbe } from "./idle-local-work.js";
|
||||
import type { ChatEndpointRuntime } from "./voice/voice-runtime.js";
|
||||
|
||||
import { fastResponseService, fastResponseSourceCurrent } from "./fast-responses.js";
|
||||
@@ -517,6 +518,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
|
||||
version: string;
|
||||
}
|
||||
>();
|
||||
// SDK initialize() and the final authority read continue after the registry's
|
||||
// replacement tail settles. Neither that work nor a failed retirement may
|
||||
// disappear just because the archived endpoint is no longer returned by get().
|
||||
const unregisterRuntimeIdleProbe = registerIdleLocalWorkProbe(() =>
|
||||
runtimeInitializations.size > 0 || runtime.hasUnsettledLifecycle() ? "present" : "none");
|
||||
const { recordCurrentMicrosoftTeamsRoute } = createTeamsRoutes(db, { runtime, runtimeCallbackEndpoint });
|
||||
const persistence = createChatSdkStatePersistence(db);
|
||||
const fetchImpl = options.fetch ?? globalThis.fetch;
|
||||
@@ -3622,12 +3628,17 @@ return {
|
||||
// Stop every provider runtime before releasing leader rows. This gives a
|
||||
// standby an immediate takeover path without overlapping Gateway sockets.
|
||||
await runtime.shutdown();
|
||||
await Promise.allSettled([...runtimeInitializations.values()].map(initialization => initialization.promise));
|
||||
await slackRegistration.close();
|
||||
await Promise.all(
|
||||
ownedDiscordGateways.map((ownership) =>
|
||||
releaseDiscordGatewayLeaseRow(ownership),
|
||||
),
|
||||
);
|
||||
if (runtimeInitializations.size > 0 || runtime.hasUnsettledLifecycle()) {
|
||||
throw new Error("Chat runtime lifecycle is still unsettled after shutdown");
|
||||
}
|
||||
unregisterRuntimeIdleProbe();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -23,6 +23,11 @@ export class ChatRuntime {
|
||||
return [...this.endpoints.values()];
|
||||
}
|
||||
|
||||
/** Retirement failures retain their owner even after get/list hide it. */
|
||||
hasUnsettledLifecycle(): boolean {
|
||||
return this.retiringEndpoints.size > 0 || this.lifecycleTails.size > 0;
|
||||
}
|
||||
|
||||
private enqueueLifecycle<T>(
|
||||
endpointId: string,
|
||||
operation: () => Promise<T>,
|
||||
|
||||
@@ -241,6 +241,8 @@ import {
|
||||
type CreateChatSdkEndpointRuntimeOptions,
|
||||
} from "./chat-sdk-runtime.js";
|
||||
|
||||
import { createChatRuntime } from "./chat-runtime.js";
|
||||
|
||||
const persistence: ChatSdkStatePersistence = {
|
||||
async compareAndSet() {
|
||||
return true;
|
||||
@@ -1152,6 +1154,31 @@ describe("Chat SDK endpoint runtime", () => {
|
||||
await registry.shutdown();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "SDK", create: createChatSdkRuntime },
|
||||
{ name: "shared", create: createChatRuntime },
|
||||
])("keeps $name lifecycle work visible while hidden retirement fails, until its owner retries", async ({ create }) => {
|
||||
const registry = create();
|
||||
const first = await registry.replaceEndpoint(registryOptions());
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(false);
|
||||
const held = deferred();
|
||||
const shutdown = vi.spyOn(first, "shutdown").mockReturnValueOnce(held.promise).mockResolvedValue(undefined);
|
||||
const removal = registry.removeEndpoint("endpoint-1");
|
||||
const result = removal.catch((error: Error) => error.message);
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(true);
|
||||
await vi.waitFor(() => expect(shutdown).toHaveBeenCalledOnce());
|
||||
expect(registry.get("endpoint-1")).toBeNull();
|
||||
expect(registry.list()).toEqual([]);
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(true);
|
||||
held.reject(new Error("retirement uncertain"));
|
||||
expect(await result).toBe("retirement uncertain");
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(true);
|
||||
await registry.removeEndpoint("endpoint-1");
|
||||
expect(shutdown).toHaveBeenCalledTimes(2);
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(false);
|
||||
await registry.shutdown();
|
||||
});
|
||||
|
||||
it("retains failed retirement ownership and only constructs a replacement after an explicit successful retry", async () => {
|
||||
const registry = createChatSdkRuntime();
|
||||
const first = await registry.replaceEndpoint(registryOptions());
|
||||
|
||||
@@ -3195,6 +3195,11 @@ export class ChatSdkRuntime {
|
||||
return [...this.endpoints.values()];
|
||||
}
|
||||
|
||||
/** Retirement failures retain their owner even after get/list hide it. */
|
||||
hasUnsettledLifecycle(): boolean {
|
||||
return this.retiringEndpoints.size > 0 || this.lifecycleTails.size > 0;
|
||||
}
|
||||
|
||||
private enqueueLifecycle<T>(
|
||||
endpointId: string,
|
||||
operation: () => Promise<T>,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { parseIdleCapabilities, type IdleCapabilities } from "./idle-incoming-wake-capability.js";
|
||||
import { createPublicKey, timingSafeEqual, verify, type JsonWebKey } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -491,6 +492,7 @@ export type CloudControlClaims = {
|
||||
sub: string;
|
||||
action: CloudControlAction;
|
||||
requestId: string;
|
||||
idleCapabilities?: IdleCapabilities;
|
||||
iat: number;
|
||||
exp: number;
|
||||
};
|
||||
@@ -572,6 +574,10 @@ export function verifyCloudControlAssertion(input: {
|
||||
) {
|
||||
throw new Error("Cloud control assertion is expired or has an invalid lifetime");
|
||||
}
|
||||
if (payload.idleCapabilities !== undefined) {
|
||||
if (payload.action !== "task-drain:read") throw new Error("Idle capabilities require task-drain:read");
|
||||
payload.idleCapabilities = parseIdleCapabilities(payload.idleCapabilities);
|
||||
}
|
||||
// Consumed LAST, only after every other check passed: a rejected
|
||||
// assertion must not burn its request id, or an attacker could deny a
|
||||
// legitimate call by replaying a mangled copy of it first.
|
||||
|
||||
@@ -10,3 +10,6 @@ export function wakeConnectionEvents(owner: object): boolean {
|
||||
wake();
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Inspection never emits a wake or polls the broker. */
|
||||
export const hasConnectionEventWakeReceiver = (owner: object): boolean => receivers.has(owner);
|
||||
@@ -146,7 +146,7 @@ function normalizeLeasedPayload(event: LeasedEvent): Record<string, unknown> {
|
||||
return compact(base);
|
||||
}
|
||||
|
||||
function bindingRows(rows: Array<{
|
||||
export function githubConnectionEventBindings(rows: Array<{
|
||||
grant: typeof connectionGrants.$inferSelect;
|
||||
connection: typeof toolConnections.$inferSelect;
|
||||
}>): GitHubBinding[] {
|
||||
@@ -259,7 +259,7 @@ export function githubConnectionEventService(
|
||||
eq(toolConnections.enabled, true),
|
||||
sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status <> 'archived')`,
|
||||
));
|
||||
return bindingRows(rows);
|
||||
return githubConnectionEventBindings(rows);
|
||||
}
|
||||
|
||||
async function applyPullRequestEvent(database: Db, companyId: string, event: LeasedEvent, effects: Array<(committedDb: Db) => Promise<void>>) {
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { IdleApiActivity } from "./idle-api-activity.js";
|
||||
|
||||
describe("authenticated API idle activity", () => {
|
||||
it("is passive without successful authentication and expires without timers or dependency work", () => {
|
||||
let monotonic = 1;
|
||||
const activity = new IdleApiActivity(() => monotonic);
|
||||
expect(activity.active()).toBe(false);
|
||||
activity.record();
|
||||
expect(activity.active()).toBe(true);
|
||||
monotonic += 599_999;
|
||||
expect(activity.active()).toBe(true);
|
||||
monotonic++;
|
||||
expect(activity.active()).toBe(false);
|
||||
});
|
||||
it("extends only from the actual latest successful use and ignores wall clock changes", () => {
|
||||
let monotonic = 1;
|
||||
const activity = new IdleApiActivity(() => monotonic);
|
||||
activity.record();
|
||||
monotonic += 300_000;
|
||||
activity.record();
|
||||
monotonic += 300_001;
|
||||
expect(activity.active()).toBe(true);
|
||||
monotonic += 299_999;
|
||||
expect(activity.active()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
/** Authenticated API use is real activity; rejecting an unknown key is not. */
|
||||
export class IdleApiActivity {
|
||||
private until = 0;
|
||||
constructor(private readonly now: () => number = () => performance.now()) {}
|
||||
record(): void {
|
||||
this.until = Math.max(this.until, this.now() + 10 * 60_000);
|
||||
}
|
||||
active(): boolean {
|
||||
return this.now() < this.until;
|
||||
}
|
||||
}
|
||||
export const idleApiActivity = new IdleApiActivity();
|
||||
@@ -0,0 +1,33 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { apiWakeFingerprint, createApiWakeAdmission, parseApiWakeAdmission, matchesApiWakeAdmission } from "./idle-api-wake-admission.js";
|
||||
|
||||
const token = "api-wake-vector-token";
|
||||
const keyHash = createHash("sha256").update(token).digest("hex");
|
||||
const input = { stackId: "vector-stack", ownerId: "00000000-0000-4000-8000-000000000001", capabilityGeneration: "a".repeat(64), challenge: "c".repeat(64) };
|
||||
const fingerprint = apiWakeFingerprint({ ...input, keyHash });
|
||||
const receipt = createApiWakeAdmission({ ...input, keys: [{ fingerprint, expiresAt: null }] });
|
||||
describe("API wake receipt wire vectors", () => {
|
||||
it("uses exact domain-separated vectors and exports neither token nor database hash", () => {
|
||||
expect(fingerprint).toBe("6478041b90f4665eeca8403186fefedc456bf1322f38dab9fb89212eecaafd43");
|
||||
expect(JSON.stringify(receipt)).not.toContain(token);
|
||||
expect(JSON.stringify(receipt)).not.toContain(keyHash);
|
||||
expect(matchesApiWakeAdmission(receipt, { stackId: input.stackId, token, now: new Date("2099-01-01").getTime() })).toBe(true);
|
||||
expect(matchesApiWakeAdmission(receipt, { stackId: "other", token, now: 0 })).toBe(false);
|
||||
expect(matchesApiWakeAdmission(receipt, { stackId: input.stackId, token: "unknown", now: 0 })).toBe(false);
|
||||
});
|
||||
it("enforces board expiry and canonical unique bounded complete receipts", () => {
|
||||
const changedChallenge = createApiWakeAdmission({ ...input, challenge: "b".repeat(64), keys: receipt.keys });
|
||||
expect(matchesApiWakeAdmission(changedChallenge, { stackId: input.stackId, token, now: 0 })).toBe(false);
|
||||
const expiring = createApiWakeAdmission({ ...input, keys: [{ fingerprint, expiresAt: new Date(1000).toISOString() }] });
|
||||
expect(matchesApiWakeAdmission(expiring, { stackId: input.stackId, token, now: 999 })).toBe(true);
|
||||
expect(matchesApiWakeAdmission(expiring, { stackId: input.stackId, token, now: 1000 })).toBe(false);
|
||||
for (const invalid of [
|
||||
{ ...receipt, generation: "b".repeat(64) }, { ...receipt, challenge: "b".repeat(64) },
|
||||
{ ...receipt, extra: true }, { ...receipt, keys: [...receipt.keys, ...receipt.keys] },
|
||||
{ ...receipt, keys: Array.from({ length: 1001 }, () => receipt.keys[0]) },
|
||||
{ ...receipt, keys: [{ fingerprint, expiresAt: "2099-01-01" }] },
|
||||
{ ...receipt, keys: [{ fingerprint: "d".repeat(64), expiresAt: null }, { fingerprint: "a".repeat(64), expiresAt: null }] },
|
||||
]) expect(() => parseApiWakeAdmission(invalid)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
export type ApiWakeClaim = { generation: string; fingerprint: string };
|
||||
export interface ApiWakeAdmission {
|
||||
version: 1;
|
||||
stackId: string;
|
||||
ownerId: string;
|
||||
capabilityGeneration: string;
|
||||
challenge: string;
|
||||
generation: string;
|
||||
keys: { fingerprint: string; expiresAt: string | null }[];
|
||||
}
|
||||
const digest = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
|
||||
const object = (value: unknown): value is Record<string, unknown> => Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||
const exact = (value: Record<string, unknown>, keys: string[]) => Object.keys(value).length === keys.length && Object.keys(value).every(key => keys.includes(key));
|
||||
const generation = (value: Omit<ApiWakeAdmission, "generation">) => createHash("sha256").update(JSON.stringify([
|
||||
"paperclip-api-wake-receipt-v1", value.version, value.stackId, value.ownerId, value.capabilityGeneration, value.challenge,
|
||||
value.keys.map(key => [key.fingerprint, key.expiresAt]),
|
||||
])).digest("hex");
|
||||
export function apiWakeFingerprint(input: { stackId: string; ownerId: string; challenge: string; keyHash: string }): string {
|
||||
if (!digest(input.keyHash) || !digest(input.challenge)) throw new Error("Invalid API wake fingerprint input");
|
||||
return createHash("sha256").update(JSON.stringify(["paperclip-api-wake-v1", input.stackId, input.ownerId, input.challenge, input.keyHash])).digest("hex");
|
||||
}
|
||||
/** Strict bounded canonical wire contract. Parsing never grants tenant permissions. */
|
||||
export function parseApiWakeAdmission(value: unknown): ApiWakeAdmission {
|
||||
if (!object(value) || !exact(value, ["version", "stackId", "ownerId", "capabilityGeneration", "challenge", "generation", "keys"])
|
||||
|| value.version !== 1 || typeof value.stackId !== "string" || !/^[A-Za-z0-9_-]{1,200}$/.test(value.stackId)
|
||||
|| typeof value.ownerId !== "string" || !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value.ownerId)
|
||||
|| !digest(value.capabilityGeneration) || !digest(value.challenge) || !digest(value.generation)
|
||||
|| !Array.isArray(value.keys) || value.keys.length > 1000) throw new Error("Invalid API wake receipt");
|
||||
let previous = "";
|
||||
for (const key of value.keys) {
|
||||
if (!object(key) || !exact(key, ["fingerprint", "expiresAt"]) || !digest(key.fingerprint) || key.fingerprint <= previous
|
||||
|| !(key.expiresAt === null || typeof key.expiresAt === "string" && key.expiresAt.length <= 30 && Number.isFinite(Date.parse(key.expiresAt))
|
||||
&& new Date(key.expiresAt).toISOString() === key.expiresAt)) throw new Error("Invalid API wake receipt key");
|
||||
previous = key.fingerprint;
|
||||
}
|
||||
const parsed = value as unknown as ApiWakeAdmission;
|
||||
if (generation(parsed) !== parsed.generation) throw new Error("API wake receipt generation changed");
|
||||
return { version: 1, stackId: parsed.stackId, ownerId: parsed.ownerId, capabilityGeneration: parsed.capabilityGeneration,
|
||||
challenge: parsed.challenge, generation: parsed.generation,
|
||||
keys: parsed.keys.map(key => ({ fingerprint: key.fingerprint, expiresAt: key.expiresAt })) };
|
||||
}
|
||||
export function createApiWakeAdmission(input: Omit<ApiWakeAdmission, "version" | "generation">): ApiWakeAdmission {
|
||||
const canonical = { version: 1 as const, ...input, keys: [...input.keys].sort((a, b) => a.fingerprint.localeCompare(b.fingerprint)) };
|
||||
return parseApiWakeAdmission({ ...canonical, generation: generation(canonical) });
|
||||
}
|
||||
export function authorizeApiWakeAdmission(receipt: ApiWakeAdmission, input: { stackId: string; token: string; now: number }): ApiWakeClaim | undefined {
|
||||
if (receipt.stackId !== input.stackId || input.token.length < 1 || input.token.length > 8192 || /\s/.test(input.token)) return;
|
||||
const fingerprint = apiWakeFingerprint({ ...receipt, keyHash: createHash("sha256").update(input.token).digest("hex") });
|
||||
return receipt.keys.some(key => key.fingerprint === fingerprint && (key.expiresAt === null || Date.parse(key.expiresAt) > input.now))
|
||||
? { generation: receipt.generation, fingerprint } : undefined;
|
||||
}
|
||||
export function matchesApiWakeAdmission(receipt: ApiWakeAdmission, input: { stackId: string; token: string; now: number }): boolean {
|
||||
return authorizeApiWakeAdmission(receipt, input) !== undefined;
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import { generateKeyPairSync, sign } from "node:crypto";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseIdleCapabilities } from "./idle-incoming-wake-capability.js";
|
||||
import {
|
||||
verifyCloudControlAssertion,
|
||||
resetCloudControlReplayFenceForTests,
|
||||
CLOUD_CONTROL_AUDIENCE,
|
||||
CLOUD_CONTROL_JWS_TYPE,
|
||||
CLOUD_RUNTIME_IDENTITY_ISSUER,
|
||||
} from "./cloud-runtime-identity.js";
|
||||
const incoming = {
|
||||
version: 1 as const,
|
||||
ownerId: "00000000-0000-4000-8000-000000000001",
|
||||
generation: "a".repeat(64),
|
||||
apiKeyProof: { version: 1 as const, challenge: "c".repeat(64) },
|
||||
routes: ["tenant-api-http-v2" as const],
|
||||
};
|
||||
const invalid: unknown[] = [
|
||||
null,
|
||||
{},
|
||||
{ other: true },
|
||||
{ deadlineCatalog: "other" },
|
||||
{ incomingWake: { ...incoming, version: 2 } },
|
||||
{ incomingWake: { ...incoming, routes: ["all"] } },
|
||||
{
|
||||
incomingWake: {
|
||||
...incoming,
|
||||
routes: ["tenant-api-http-v2", "tenant-api-http-v2"],
|
||||
},
|
||||
},
|
||||
{ incomingWake: { ...incoming, ownerId: "other" } },
|
||||
{ incomingWake: { ...incoming, extra: true } },
|
||||
{
|
||||
incomingWake: {
|
||||
...incoming,
|
||||
githubBroker: {
|
||||
instanceId: "instance",
|
||||
bindingsFingerprint: "b".repeat(64),
|
||||
},
|
||||
},
|
||||
},
|
||||
{ incomingWake: { ...incoming, routes: ["github-broker-events-v1"] } },
|
||||
];
|
||||
|
||||
describe("signed idle capabilities", () => {
|
||||
it.each(invalid)("rejects unknown/partial capabilities: %j", (value) => {
|
||||
expect(() => parseIdleCapabilities(value)).toThrow();
|
||||
});
|
||||
it("detaches and freezes the exact known catalogs", () => {
|
||||
const source = {
|
||||
incomingWake: { ...incoming, routes: [...incoming.routes] },
|
||||
deadlineCatalog: "domain_deadlines_v1",
|
||||
};
|
||||
const parsed = parseIdleCapabilities(source);
|
||||
source.incomingWake.routes.length = 0;
|
||||
expect(parsed.incomingWake?.routes).toEqual(["tenant-api-http-v2"]);
|
||||
expect(Object.isFrozen(parsed.incomingWake?.routes)).toBe(true);
|
||||
});
|
||||
it("binds capability to signed stack, action and replay fence", () => {
|
||||
resetCloudControlReplayFenceForTests();
|
||||
const pair = generateKeyPairSync("ed25519");
|
||||
const now = new Date("2099-01-01T00:00:00Z");
|
||||
const jwk = {
|
||||
...pair.publicKey.export({ format: "jwk" }),
|
||||
kid: "idle",
|
||||
use: "sig",
|
||||
alg: "EdDSA",
|
||||
};
|
||||
const env = {
|
||||
PAPERCLIP_CLOUD_STACK_ID: "stack",
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [jwk] }),
|
||||
};
|
||||
const encode = (v: unknown) =>
|
||||
Buffer.from(JSON.stringify(v)).toString("base64url");
|
||||
const token = (overrides: object = {}) => {
|
||||
const header = encode({
|
||||
alg: "EdDSA",
|
||||
typ: CLOUD_CONTROL_JWS_TYPE,
|
||||
kid: "idle",
|
||||
});
|
||||
const body = encode({
|
||||
v: 1,
|
||||
iss: CLOUD_RUNTIME_IDENTITY_ISSUER,
|
||||
aud: CLOUD_CONTROL_AUDIENCE,
|
||||
sub: "stack",
|
||||
action: "task-drain:read",
|
||||
requestId: "same",
|
||||
iat: now.getTime() / 1000,
|
||||
exp: now.getTime() / 1000 + 60,
|
||||
idleCapabilities: {
|
||||
incomingWake: incoming,
|
||||
deadlineCatalog: "domain_deadlines_v1",
|
||||
},
|
||||
...overrides,
|
||||
});
|
||||
return `${header}.${body}.${sign(null, Buffer.from(`${header}.${body}`), pair.privateKey).toString("base64url")}`;
|
||||
};
|
||||
const verify = (compactJws: string) =>
|
||||
verifyCloudControlAssertion({
|
||||
compactJws,
|
||||
env,
|
||||
now,
|
||||
expectedAction: "task-drain:read",
|
||||
});
|
||||
expect(() => verify(token({ sub: "foreign" }))).toThrow();
|
||||
expect(() => verify(token({ action: "task-drain:start" }))).toThrow();
|
||||
expect(() =>
|
||||
verify(token({ idleCapabilities: { deadlineCatalog: "unknown" } })),
|
||||
).toThrow();
|
||||
expect(verify(token()).idleCapabilities?.incomingWake).toEqual(incoming);
|
||||
expect(() => verify(token())).toThrow(/already been used/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
/** Closed Cloud-owned ingress catalog. No credential material belongs in this proof. */
|
||||
export interface IdleIncomingWakeCapability {
|
||||
version: 1;
|
||||
ownerId: string;
|
||||
generation: string;
|
||||
routes: ("tenant-api-http-v2" | "github-broker-events-v1")[];
|
||||
apiKeyProof?: { version: 1; challenge: string };
|
||||
githubBroker?: { instanceId: string; bindingsFingerprint: string };
|
||||
}
|
||||
export interface IdleCapabilities {
|
||||
incomingWake?: IdleIncomingWakeCapability;
|
||||
deadlineCatalog?: "domain_deadlines_v1";
|
||||
}
|
||||
const object = (v: unknown): v is Record<string, unknown> =>
|
||||
Boolean(v && typeof v === "object" && !Array.isArray(v));
|
||||
const exact = (v: Record<string, unknown>, keys: string[]) =>
|
||||
Object.keys(v).every((k) => keys.includes(k));
|
||||
const digest = (v: unknown): v is string =>
|
||||
typeof v === "string" && /^[a-f0-9]{64}$/.test(v);
|
||||
export function parseIdleCapabilities(value: unknown): IdleCapabilities {
|
||||
if (
|
||||
!object(value) ||
|
||||
!exact(value, ["incomingWake", "deadlineCatalog"]) ||
|
||||
!Object.keys(value).length
|
||||
)
|
||||
throw new Error("Invalid idle capabilities");
|
||||
if (
|
||||
value.deadlineCatalog !== undefined &&
|
||||
value.deadlineCatalog !== "domain_deadlines_v1"
|
||||
)
|
||||
throw new Error("Unknown idle deadline catalog");
|
||||
if (value.incomingWake !== undefined) {
|
||||
const v = value.incomingWake;
|
||||
if (
|
||||
!object(v) ||
|
||||
!exact(v, [
|
||||
"version",
|
||||
"ownerId",
|
||||
"generation",
|
||||
"routes",
|
||||
"githubBroker",
|
||||
"apiKeyProof",
|
||||
]) ||
|
||||
v.version !== 1 ||
|
||||
typeof v.ownerId !== "string" ||
|
||||
!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(
|
||||
v.ownerId,
|
||||
) ||
|
||||
!digest(v.generation) ||
|
||||
!Array.isArray(v.routes) ||
|
||||
v.routes.length < 1 ||
|
||||
v.routes.length > 2 ||
|
||||
v.routes.some(
|
||||
(r) => !["tenant-api-http-v2", "github-broker-events-v1"].includes(r),
|
||||
) ||
|
||||
new Set(v.routes).size !== v.routes.length ||
|
||||
[...v.routes].sort().join() !== v.routes.join()
|
||||
)
|
||||
throw new Error("Invalid incoming wake capability");
|
||||
if (v.routes.includes("tenant-api-http-v2")) {
|
||||
const proof = v.apiKeyProof;
|
||||
if (!object(proof) || Object.keys(proof).length !== 2 || !exact(proof, ["version", "challenge"])
|
||||
|| proof.version !== 1 || !digest(proof.challenge)) throw new Error("Invalid API wake challenge");
|
||||
} else if (v.apiKeyProof !== undefined) throw new Error("Unexpected API wake challenge");
|
||||
if (v.routes.includes("github-broker-events-v1")) {
|
||||
const b = v.githubBroker;
|
||||
if (
|
||||
!object(b) ||
|
||||
!exact(b, ["instanceId", "bindingsFingerprint"]) ||
|
||||
typeof b.instanceId !== "string" ||
|
||||
!/^[A-Za-z0-9_-]{1,200}$/.test(b.instanceId) ||
|
||||
!digest(b.bindingsFingerprint)
|
||||
)
|
||||
throw new Error("Invalid GitHub wake binding");
|
||||
} else if (v.githubBroker !== undefined)
|
||||
throw new Error("Unexpected GitHub wake binding");
|
||||
}
|
||||
// Return a detached immutable snapshot; request callers cannot mutate a verified claim.
|
||||
const parsed = JSON.parse(JSON.stringify(value)) as IdleCapabilities;
|
||||
if (parsed.incomingWake) {
|
||||
Object.freeze(parsed.incomingWake.routes);
|
||||
if (parsed.incomingWake.apiKeyProof) Object.freeze(parsed.incomingWake.apiKeyProof);
|
||||
if (parsed.incomingWake.githubBroker)
|
||||
Object.freeze(parsed.incomingWake.githubBroker);
|
||||
Object.freeze(parsed.incomingWake);
|
||||
}
|
||||
return Object.freeze(parsed);
|
||||
}
|
||||
|
||||
declare const verified: unique symbol;
|
||||
export type VerifiedIdleCapabilities = IdleCapabilities & {
|
||||
readonly [verified]: true;
|
||||
};
|
||||
@@ -0,0 +1,184 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
readIdleIncomingWakeProof,
|
||||
incomingWakeFingerprint,
|
||||
} from "./idle-incoming-wake.js";
|
||||
import { registerConnectionEventWake } from "./connection-event-wake.js";
|
||||
import type { VerifiedIdleCapabilities } from "./idle-incoming-wake-capability.js";
|
||||
|
||||
const ownerId = "00000000-0000-4000-8000-000000000001";
|
||||
function fixture(count: number) {
|
||||
const endpoints = Array.from({ length: count }, (_, i) => ({
|
||||
endpoint: {
|
||||
id: `endpoint-${i}`,
|
||||
companyId: "company",
|
||||
status: "active",
|
||||
activatedAt: new Date(0),
|
||||
botExternalId: `bot-${i}`,
|
||||
providerAccountId: `app-${i}`,
|
||||
setup: {
|
||||
step: "complete",
|
||||
github: {
|
||||
cloudRegistrationId: String(i).padStart(43, "a"),
|
||||
registrationStatus: "completed",
|
||||
},
|
||||
},
|
||||
},
|
||||
connection: { id: `connection-${i}` },
|
||||
}));
|
||||
const registrations = endpoints.map(({ endpoint }) => ({
|
||||
endpointId: endpoint.id,
|
||||
companyId: endpoint.companyId,
|
||||
userId: "owner",
|
||||
cloudId: endpoint.setup.github.cloudRegistrationId,
|
||||
}));
|
||||
const tuples = registrations.map((row) => [
|
||||
`github-app:${row.cloudId}`,
|
||||
row.companyId,
|
||||
row.userId,
|
||||
"github.bot",
|
||||
`gateway:${row.cloudId}`,
|
||||
row.cloudId,
|
||||
row.cloudId,
|
||||
]);
|
||||
const capability = {
|
||||
incomingWake: {
|
||||
version: 1,
|
||||
ownerId,
|
||||
generation: "a".repeat(64),
|
||||
routes: ["github-broker-events-v1"],
|
||||
githubBroker: {
|
||||
instanceId: "fixture-instance",
|
||||
bindingsFingerprint: incomingWakeFingerprint(tuples),
|
||||
},
|
||||
},
|
||||
} as VerifiedIdleCapabilities;
|
||||
return { endpoints, registrations, capability };
|
||||
}
|
||||
/** Named DB boundary for cardinality/elapsed guards; PostgreSQL fixture covers real SQL separately. */
|
||||
function database(replies: Array<() => unknown[]>) {
|
||||
let index = 0;
|
||||
const select = vi.fn(() => {
|
||||
const reply = replies[index++];
|
||||
const query = {
|
||||
from: () => query,
|
||||
innerJoin: () => query,
|
||||
where: () => query,
|
||||
limit: async () => reply?.() ?? [],
|
||||
};
|
||||
return query;
|
||||
});
|
||||
const execute = vi.fn(async () => []);
|
||||
return {
|
||||
db: { select, execute } as unknown as Pick<Db, "select" | "execute">,
|
||||
select,
|
||||
execute,
|
||||
};
|
||||
}
|
||||
const stops: Array<() => void> = [];
|
||||
function input(
|
||||
db: object,
|
||||
capabilities: VerifiedIdleCapabilities,
|
||||
now?: () => number,
|
||||
) {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "fixture-instance");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", "fixture");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", "fixture");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "http://127.0.0.1");
|
||||
stops.push(registerConnectionEventWake(db, () => {}));
|
||||
return { ownerId, receiverOwner: db, capabilities, now };
|
||||
}
|
||||
afterEach(() => {
|
||||
for (const stop of stops.splice(0)) stop();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe("bounded incoming wake owner projection", () => {
|
||||
it("uses three bounded selects for a thousand bots and retains exact per-bot authority", async () => {
|
||||
const f = fixture(1000),
|
||||
sql = database([() => [], () => f.endpoints, () => f.registrations]);
|
||||
const result = await readIdleIncomingWakeProof(
|
||||
sql.db,
|
||||
input(sql.db, f.capability),
|
||||
);
|
||||
expect(sql.select).toHaveBeenCalledTimes(3);
|
||||
expect(sql.execute).toHaveBeenCalledTimes(4);
|
||||
expect(result?.endpointIds).toHaveLength(1000);
|
||||
expect(result?.connectionIds).toHaveLength(1000);
|
||||
});
|
||||
it.each(["duplicate", "foreign", "missing"])(
|
||||
"rejects %s registration authority",
|
||||
async (fault) => {
|
||||
const f = fixture(1);
|
||||
let registrations = f.registrations;
|
||||
if (fault === "duplicate")
|
||||
registrations = [...registrations, ...registrations];
|
||||
if (fault === "foreign")
|
||||
registrations = registrations.map((r) => ({
|
||||
...r,
|
||||
companyId: "foreign",
|
||||
}));
|
||||
if (fault === "missing") registrations = [];
|
||||
const sql = database([() => [], () => f.endpoints, () => registrations]);
|
||||
await expect(
|
||||
readIdleIncomingWakeProof(sql.db, input(sql.db, f.capability)),
|
||||
).rejects.toThrow("registration authority");
|
||||
},
|
||||
);
|
||||
it("stops the complete inspection when its monotonic budget expires before another query", async () => {
|
||||
const f = fixture(1);
|
||||
let now = 0;
|
||||
const sql = database([
|
||||
() => {
|
||||
now = 2001;
|
||||
return [];
|
||||
},
|
||||
() => f.endpoints,
|
||||
() => f.registrations,
|
||||
]);
|
||||
await expect(
|
||||
readIdleIncomingWakeProof(
|
||||
sql.db,
|
||||
input(sql.db, f.capability, () => now),
|
||||
),
|
||||
).rejects.toThrow("budget exhausted");
|
||||
expect(sql.select).toHaveBeenCalledTimes(1);
|
||||
expect(sql.execute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it("fails closed at row limits before expanding work or issuing registration queries", async () => {
|
||||
const f = fixture(1001),
|
||||
sql = database([() => [], () => f.endpoints, () => f.registrations]);
|
||||
await expect(
|
||||
readIdleIncomingWakeProof(sql.db, input(sql.db, f.capability)),
|
||||
).rejects.toThrow("Too many GitHub endpoints");
|
||||
expect(sql.select).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("bounded API admission projection", () => {
|
||||
const api = { incomingWake: { version: 1, ownerId, generation: "a".repeat(64), routes: ["tenant-api-http-v2"],
|
||||
apiKeyProof: { version: 1, challenge: "c".repeat(64) } } } as VerifiedIdleCapabilities;
|
||||
it.each(["overflow", "duplicate", "malformed"])("refuses complete eligibility for %s key state", async kind => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack");
|
||||
const row = { keyHash: "a".repeat(64), expiresAt: null };
|
||||
const rows = kind === "overflow" ? Array.from({ length: 1001 }, () => row)
|
||||
: kind === "duplicate" ? [row, row] : [{ keyHash: "not-a-digest", expiresAt: null }];
|
||||
const execute = vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce(rows);
|
||||
await expect(readIdleIncomingWakeProof({ execute } as unknown as Pick<Db, "select" | "execute">,
|
||||
{ ownerId, receiverOwner: {}, capabilities: api })).rejects.toThrow();
|
||||
expect(execute).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("has an explicit complete empty-key receipt and shares the inspection time budget", async () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack");
|
||||
const execute = vi.fn(async () => []);
|
||||
const proof = await readIdleIncomingWakeProof({ execute } as unknown as Pick<Db, "select" | "execute">,
|
||||
{ ownerId, receiverOwner: {}, capabilities: api });
|
||||
expect(proof?.apiWakeAdmission?.keys).toEqual([]);
|
||||
let clock = 0;
|
||||
const slow = vi.fn(async () => { clock += 1100; return []; });
|
||||
await expect(readIdleIncomingWakeProof({ execute: slow } as unknown as Pick<Db, "select" | "execute">,
|
||||
{ ownerId, receiverOwner: {}, capabilities: api, now: () => clock })).rejects.toThrow(/budget exhausted/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,359 @@
|
||||
import { apiWakeFingerprint, createApiWakeAdmission, type ApiWakeAdmission } from "./idle-api-wake-admission.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq, inArray, sql, type SQL } from "drizzle-orm";
|
||||
import {
|
||||
chatEndpoints,
|
||||
chatGitHubRegistrations,
|
||||
connectionGrants,
|
||||
toolConnections,
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import type {
|
||||
IdleIncomingWakeCapability,
|
||||
VerifiedIdleCapabilities,
|
||||
} from "./idle-incoming-wake-capability.js";
|
||||
import { githubConnectionEventBindings } from "./github-connection-events.js";
|
||||
import { paperclipCloudConnectorConfigFromEnv } from "./paperclip-cloud-connector.js";
|
||||
import { hasConnectionEventWakeReceiver } from "./connection-event-wake.js";
|
||||
|
||||
export interface IdleIncomingWakeProof {
|
||||
capability: IdleIncomingWakeCapability;
|
||||
apiWakeAdmission?: ApiWakeAdmission;
|
||||
connectionIds: string[];
|
||||
endpointIds: string[];
|
||||
/** A broker enrollment cannot discharge these credential continuity obligations. */
|
||||
requiredContinuityConnectionIds: string[];
|
||||
}
|
||||
export const incomingWakeFingerprint = (bindings: string[][]) =>
|
||||
createHash("sha256")
|
||||
.update(
|
||||
JSON.stringify(
|
||||
bindings
|
||||
.map((b) => [...b])
|
||||
.sort((a, b) =>
|
||||
JSON.stringify(a) < JSON.stringify(b)
|
||||
? -1
|
||||
: JSON.stringify(a) > JSON.stringify(b)
|
||||
? 1
|
||||
: 0,
|
||||
),
|
||||
),
|
||||
)
|
||||
.digest("hex");
|
||||
|
||||
/** Read current binding authority in the safety transaction. No network call, renewal or write. */
|
||||
export async function readIdleIncomingWakeProof(
|
||||
tx: Pick<Db, "select" | "execute">,
|
||||
input: {
|
||||
capabilities?: VerifiedIdleCapabilities;
|
||||
ownerId: string;
|
||||
receiverOwner: object;
|
||||
now?: () => number;
|
||||
},
|
||||
): Promise<IdleIncomingWakeProof | undefined> {
|
||||
const capability = input.capabilities?.incomingWake;
|
||||
if (!capability) return;
|
||||
if (capability.ownerId !== input.ownerId)
|
||||
throw new Error("Incoming wake owner changed");
|
||||
const proof: IdleIncomingWakeProof = {
|
||||
capability,
|
||||
connectionIds: [],
|
||||
endpointIds: [],
|
||||
requiredContinuityConnectionIds: [],
|
||||
};
|
||||
const monotonic = input.now ?? (() => performance.now());
|
||||
const started = monotonic();
|
||||
const remaining = () => {
|
||||
const ms = 2_000 - (monotonic() - started);
|
||||
if (!Number.isFinite(ms) || ms <= 0)
|
||||
throw new Error("Incoming wake inspection budget exhausted");
|
||||
return Math.max(1, Math.floor(ms));
|
||||
};
|
||||
const beforeRead = async () => {
|
||||
// One shared owner budget, not five seconds multiplied by the row count.
|
||||
await tx.execute(
|
||||
sql`SELECT set_config('statement_timeout', ${`${remaining()}ms`}, true)`,
|
||||
);
|
||||
remaining();
|
||||
};
|
||||
if (capability.routes.includes("tenant-api-http-v2")) {
|
||||
const stackId = process.env.PAPERCLIP_CLOUD_STACK_ID;
|
||||
if (!stackId || !capability.apiKeyProof) throw new Error("API wake challenge is missing");
|
||||
await beforeRead();
|
||||
const rows = await tx.execute<{ keyHash: string; expiresAt: Date | null }>(sql`
|
||||
SELECT key_hash AS "keyHash", NULL::timestamptz AS "expiresAt" FROM agent_api_keys WHERE revoked_at IS NULL
|
||||
UNION ALL SELECT key_hash AS "keyHash", expires_at AS "expiresAt" FROM board_api_keys
|
||||
WHERE revoked_at IS NULL AND (expires_at IS NULL OR expires_at > now()) LIMIT 1001`);
|
||||
remaining();
|
||||
if (rows.length > 1000) throw new Error("Too many API keys to certify wake admission");
|
||||
proof.apiWakeAdmission = createApiWakeAdmission({ stackId, ownerId: input.ownerId,
|
||||
capabilityGeneration: capability.generation, challenge: capability.apiKeyProof.challenge,
|
||||
keys: rows.map(row => ({ fingerprint: apiWakeFingerprint({ stackId, ownerId: input.ownerId,
|
||||
challenge: capability.apiKeyProof!.challenge, keyHash: row.keyHash }),
|
||||
expiresAt: row.expiresAt === null ? null : new Date(row.expiresAt).toISOString() })),
|
||||
});
|
||||
}
|
||||
if (!capability.routes.includes("github-broker-events-v1")) {
|
||||
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
|
||||
remaining();
|
||||
return proof;
|
||||
}
|
||||
const broker = capability.githubBroker;
|
||||
const config = paperclipCloudConnectorConfigFromEnv();
|
||||
if (
|
||||
!broker ||
|
||||
config?.instanceId !== broker.instanceId ||
|
||||
!hasConnectionEventWakeReceiver(input.receiverOwner)
|
||||
)
|
||||
throw new Error("Incoming GitHub receiver is not bound");
|
||||
await beforeRead();
|
||||
const rows = await tx
|
||||
.select({ grant: connectionGrants, connection: toolConnections })
|
||||
.from(connectionGrants)
|
||||
.innerJoin(
|
||||
toolConnections,
|
||||
and(
|
||||
eq(toolConnections.id, connectionGrants.connectionId),
|
||||
eq(toolConnections.companyId, connectionGrants.companyId),
|
||||
),
|
||||
)
|
||||
.where(
|
||||
and(
|
||||
eq(connectionGrants.status, "active"),
|
||||
eq(toolConnections.status, "active"),
|
||||
eq(toolConnections.enabled, true),
|
||||
sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`,
|
||||
sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`,
|
||||
sql`exists (select 1 from companies where id=${connectionGrants.companyId} and status <> 'archived')`,
|
||||
),
|
||||
)
|
||||
.limit(1001);
|
||||
if (rows.length > 1000)
|
||||
throw new Error("Too many GitHub bindings to inspect");
|
||||
remaining();
|
||||
let bindingCount = 0;
|
||||
for (const { grant, connection } of rows) {
|
||||
const github = grant.providerTenant?.github;
|
||||
if (
|
||||
!github ||
|
||||
!Array.isArray(github.installationIds) ||
|
||||
!github.installationIds.length ||
|
||||
github.installationIds.length > 1000 ||
|
||||
github.bindingSync ||
|
||||
github.continuityRetry ||
|
||||
github.installationIds.some(
|
||||
(id) => typeof id !== "string" || !/^[0-9]+$/.test(id),
|
||||
)
|
||||
)
|
||||
throw new Error("Unsettled GitHub authority");
|
||||
bindingCount += github.installationIds.length;
|
||||
if (bindingCount > 1000)
|
||||
throw new Error("Too many GitHub bindings to inspect");
|
||||
proof.requiredContinuityConnectionIds.push(connection.id);
|
||||
}
|
||||
const bindings = githubConnectionEventBindings(rows);
|
||||
if (
|
||||
bindings.length !==
|
||||
rows.reduce(
|
||||
(n, r) => n + r.grant.providerTenant!.github!.installationIds.length,
|
||||
0,
|
||||
)
|
||||
)
|
||||
throw new Error("Incomplete GitHub binding");
|
||||
const tuples = bindings.map((b) => [
|
||||
b.id,
|
||||
b.companyId,
|
||||
b.subject,
|
||||
"github.code",
|
||||
b.installationId,
|
||||
b.connectionId,
|
||||
b.grantId,
|
||||
]);
|
||||
await beforeRead();
|
||||
const endpoints = await tx
|
||||
.select({ endpoint: chatEndpoints, connection: toolConnections })
|
||||
.from(chatEndpoints)
|
||||
.innerJoin(
|
||||
toolConnections,
|
||||
and(
|
||||
eq(toolConnections.id, chatEndpoints.connectionId),
|
||||
eq(toolConnections.companyId, chatEndpoints.companyId),
|
||||
),
|
||||
)
|
||||
.where(
|
||||
and(
|
||||
eq(chatEndpoints.provider, "github"),
|
||||
eq(chatEndpoints.status, "active"),
|
||||
eq(toolConnections.status, "active"),
|
||||
eq(toolConnections.enabled, true),
|
||||
eq(toolConnections.transport, "chat_sdk"),
|
||||
eq(toolConnections.connectionPurpose, "channel"),
|
||||
sql`exists (select 1 from companies where id=${chatEndpoints.companyId} and status <> 'archived')`,
|
||||
),
|
||||
)
|
||||
.limit(1001);
|
||||
if (endpoints.length > 1000)
|
||||
throw new Error("Too many GitHub endpoints to inspect");
|
||||
remaining();
|
||||
const brokerEndpointIds = endpoints
|
||||
.filter(
|
||||
({ endpoint }) =>
|
||||
typeof endpoint.setup.github?.cloudRegistrationId === "string",
|
||||
)
|
||||
.map(({ endpoint }) => endpoint.id);
|
||||
const registrationUsers = new Map<string, string[]>();
|
||||
if (brokerEndpointIds.length) {
|
||||
await beforeRead();
|
||||
const registrations = await tx
|
||||
.select({
|
||||
endpointId: chatGitHubRegistrations.endpointId,
|
||||
companyId: chatGitHubRegistrations.companyId,
|
||||
userId: chatGitHubRegistrations.userId,
|
||||
cloudId: sql<string>`${chatGitHubRegistrations.handoff}->>'cloudId'`,
|
||||
})
|
||||
.from(chatGitHubRegistrations)
|
||||
.where(
|
||||
and(
|
||||
inArray(chatGitHubRegistrations.endpointId, brokerEndpointIds),
|
||||
eq(chatGitHubRegistrations.status, "completed"),
|
||||
),
|
||||
)
|
||||
.limit(10001);
|
||||
remaining();
|
||||
if (registrations.length > 10000)
|
||||
throw new Error("Too many GitHub registrations to inspect");
|
||||
for (const registration of registrations) {
|
||||
const key = JSON.stringify([
|
||||
registration.endpointId,
|
||||
registration.companyId,
|
||||
registration.cloudId,
|
||||
]);
|
||||
const users = registrationUsers.get(key) ?? [];
|
||||
users.push(registration.userId);
|
||||
registrationUsers.set(key, users);
|
||||
}
|
||||
}
|
||||
for (const { endpoint, connection } of endpoints) {
|
||||
const registration = endpoint.setup.github?.cloudRegistrationId;
|
||||
if (!registration || !/^[A-Za-z0-9_-]{43}$/.test(registration)) continue;
|
||||
if (
|
||||
endpoint.setup.step !== "complete" ||
|
||||
endpoint.setup.github?.registrationStatus !== "completed" ||
|
||||
endpoint.setup.github.initialSetupPending ||
|
||||
endpoint.setup.github.initialRepositoryImportPending ||
|
||||
!endpoint.activatedAt ||
|
||||
!endpoint.botExternalId ||
|
||||
!endpoint.providerAccountId
|
||||
)
|
||||
throw new Error("GitHub bot setup has unfinished work");
|
||||
const users = registrationUsers.get(
|
||||
JSON.stringify([endpoint.id, endpoint.companyId, registration]),
|
||||
);
|
||||
if (users?.length !== 1 || !users[0])
|
||||
throw new Error("GitHub registration authority changed");
|
||||
tuples.push([
|
||||
`github-app:${registration}`,
|
||||
endpoint.companyId,
|
||||
users[0],
|
||||
"github.bot",
|
||||
`gateway:${registration}`,
|
||||
registration,
|
||||
registration,
|
||||
]);
|
||||
proof.connectionIds.push(connection.id);
|
||||
proof.endpointIds.push(endpoint.id);
|
||||
}
|
||||
remaining();
|
||||
if (tuples.length > 1000)
|
||||
throw new Error("Too many GitHub bindings to inspect");
|
||||
if (incomingWakeFingerprint(tuples) !== broker.bindingsFingerprint)
|
||||
throw new Error("GitHub wake bindings changed");
|
||||
remaining();
|
||||
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
|
||||
remaining();
|
||||
return proof;
|
||||
}
|
||||
|
||||
/** Exact domain predicates selected by stable inventory reason, never SQL-text identity. */
|
||||
export function incomingWakeWorkQuery(
|
||||
reason: string,
|
||||
proof?: IdleIncomingWakeProof,
|
||||
continuityConnectionIds: readonly string[] = [],
|
||||
): SQL | undefined {
|
||||
const connectionIds = [
|
||||
...(proof?.connectionIds ?? []),
|
||||
...(proof?.requiredContinuityConnectionIds.filter((id) =>
|
||||
continuityConnectionIds.includes(id),
|
||||
) ?? []),
|
||||
];
|
||||
if (reason === "agent_api_keys")
|
||||
return proof?.apiWakeAdmission !== undefined
|
||||
? sql`SELECT 1 WHERE false`
|
||||
: sql`SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`;
|
||||
if (reason === "board_api_keys")
|
||||
return proof?.apiWakeAdmission !== undefined
|
||||
? sql`SELECT 1 WHERE false`
|
||||
: sql`SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL
|
||||
AND (expires_at IS NULL OR expires_at > now())`;
|
||||
if (reason === "chat_endpoints") {
|
||||
const ids = proof?.endpointIds ?? [];
|
||||
// Only the supported GitHub webhook path gains a removed-history proof.
|
||||
// Archival commits before cleanup, so status alone cannot discharge work.
|
||||
return sql`SELECT 1 FROM chat_endpoints endpoint WHERE (((endpoint.provider = 'github'
|
||||
AND endpoint.status = 'archived' AND endpoint.archived_at IS NOT NULL
|
||||
AND jsonb_typeof(endpoint.setup) = 'object'
|
||||
AND CASE WHEN jsonb_typeof(endpoint.setup->'runtimeGeneration') = 'number'
|
||||
THEN (endpoint.setup->>'runtimeGeneration')::numeric >= 1
|
||||
AND trunc((endpoint.setup->>'runtimeGeneration')::numeric) = (endpoint.setup->>'runtimeGeneration')::numeric
|
||||
ELSE false END
|
||||
AND EXISTS (SELECT 1 FROM tool_connections removed_connection
|
||||
WHERE removed_connection.id = endpoint.connection_id AND removed_connection.company_id = endpoint.company_id
|
||||
AND removed_connection.connection_purpose = 'channel' AND removed_connection.transport = 'chat_sdk'
|
||||
AND removed_connection.status = 'archived' AND removed_connection.enabled = false
|
||||
AND removed_connection.credential_secret_refs = '[]'::jsonb)
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_endpoint_leases lease WHERE lease.endpoint_id = endpoint.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_github_registrations registration WHERE registration.endpoint_id = endpoint.id
|
||||
AND (registration.company_id = endpoint.company_id AND registration.status = 'completed'
|
||||
AND registration.consumed_at IS NOT NULL) IS NOT TRUE))
|
||||
OR (endpoint.provider = 'slack'
|
||||
AND endpoint.status = 'draft' AND endpoint.setup->>'step' = 'provider_setup'
|
||||
AND (endpoint.setup - ARRAY['step', 'command']) = '{}'::jsonb
|
||||
AND (endpoint.setup->'command' IS NULL OR endpoint.setup->'command' = 'null'::jsonb
|
||||
OR jsonb_typeof(endpoint.setup->'command') = 'string')
|
||||
AND endpoint.activated_at IS NULL AND endpoint.provider_account_id IS NULL
|
||||
AND endpoint.bot_external_id IS NULL AND endpoint.bot_username IS NULL
|
||||
AND endpoint.last_event_at IS NULL AND endpoint.last_publication_at IS NULL
|
||||
AND endpoint.health_message IS NULL AND endpoint.last_error IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id)
|
||||
AND EXISTS (SELECT 1 FROM tool_connections draft_connection
|
||||
WHERE draft_connection.id = endpoint.connection_id AND draft_connection.company_id = endpoint.company_id
|
||||
AND draft_connection.connection_purpose = 'channel' AND draft_connection.transport = 'chat_sdk'
|
||||
AND draft_connection.status = 'draft' AND draft_connection.enabled = false))) IS NOT TRUE) AND ${
|
||||
ids.length
|
||||
? sql`endpoint.id NOT IN (${sql.join(
|
||||
ids.map((id) => sql`${id}::uuid`),
|
||||
sql`, `,
|
||||
)})`
|
||||
: sql`true`
|
||||
}`;
|
||||
}
|
||||
if (reason === "tool_connections") {
|
||||
const ids = connectionIds;
|
||||
return sql`SELECT 1 FROM tool_connections c WHERE (c.status <> 'archived' AND (
|
||||
(c.status = 'draft' AND c.enabled = false)
|
||||
OR (c.status = 'active' AND c.connection_purpose = 'ai' AND c.transport = 'runtime_auth')
|
||||
OR (c.status = 'active' AND c.connection_purpose = 'tool' AND c.transport = 'mcp_remote'
|
||||
AND NOT EXISTS (SELECT 1 FROM connection_grants g WHERE g.connection_id = c.id
|
||||
AND (g.company_id <> c.company_id OR g.provider_tenant->'github' IS NOT NULL)))
|
||||
) IS NOT TRUE) AND ${
|
||||
ids.length
|
||||
? sql`c.id NOT IN (${sql.join(
|
||||
ids.map((id) => sql`${id}::uuid`),
|
||||
sql`, `,
|
||||
)})`
|
||||
: sql`true`
|
||||
}`;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { idleApiActivity } from "./idle-api-activity.js";
|
||||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
|
||||
@@ -83,7 +84,7 @@ export async function readIdleLocalWork(options: { backupCheckpoint?: boolean; d
|
||||
}
|
||||
let probes: IdleLocalWork;
|
||||
try {
|
||||
const local = [...localWorkProbes].map(probe => probe());
|
||||
const local = [idleApiActivity.active() ? "present" : "none", ...[...localWorkProbes].map(probe => probe())];
|
||||
probes = local.some(state => state !== "none" && state !== "present") ? "unknown"
|
||||
: local.includes("present") ? "present" : "none";
|
||||
} catch { probes = "unknown"; }
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import type { ApiWakeAdmission } from "./idle-api-wake-admission.js";
|
||||
import { readIdleIncomingWakeProof, incomingWakeWorkQuery, type IdleIncomingWakeProof } from "./idle-incoming-wake.js";
|
||||
import type { VerifiedIdleCapabilities, IdleIncomingWakeCapability } from "./idle-incoming-wake-capability.js";
|
||||
import { idleWorkSnapshot } from "./task-admission.js";
|
||||
import { readIdleLocalWork, type IdleLocalWork } from "./idle-local-work.js";
|
||||
import { readIdleRoutineSchedule, type ScheduledWake } from "./idle-scheduled-work.js";
|
||||
@@ -38,10 +41,12 @@ export interface IdleSleepDrainStatus {
|
||||
}
|
||||
|
||||
export interface IdleSleepSafety {
|
||||
version: 1 | 2;
|
||||
version: 1 | 2 | 3;
|
||||
backgroundWork: "none" | "present" | "unknown";
|
||||
scheduledWake?: ScheduledWake;
|
||||
diagnostics?: IdleSleepDiagnostics;
|
||||
incomingWake?: IdleIncomingWakeCapability;
|
||||
apiWakeAdmission?: ApiWakeAdmission;
|
||||
}
|
||||
|
||||
export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) => Promise<{
|
||||
@@ -84,9 +89,8 @@ const WORK_CHECKS = [
|
||||
workCheck("prompt_injection_invocations", `SELECT 1 FROM prompt_injection_invocations WHERE status = 'running'`),
|
||||
// Persistent machine credentials can receive work without a human opening
|
||||
// the instance. They need an ingress wake contract before we can sleep.
|
||||
workCheck("agent_api_keys", `SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`),
|
||||
workCheck("board_api_keys", `SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL
|
||||
AND (expires_at IS NULL OR expires_at > now())`),
|
||||
workCheck("agent_api_keys", incomingWakeWorkQuery("agent_api_keys")!),
|
||||
workCheck("board_api_keys", incomingWakeWorkQuery("board_api_keys")!),
|
||||
// Coalescing consumes a request into another run. Retain it as work until
|
||||
// both the request and that exact company/agent's run have finished.
|
||||
workCheck("agent_wakeup_requests", `SELECT 1 FROM agent_wakeup_requests w WHERE
|
||||
@@ -137,20 +141,7 @@ const WORK_CHECKS = [
|
||||
// An unconfigured Slack draft has no provider identity or inbound runtime.
|
||||
// Setup still wakes through normal browser admission. Retain every enabled,
|
||||
// previously active, partially connected, foreign, or unsupported endpoint.
|
||||
workCheck("chat_endpoints", `SELECT 1 FROM chat_endpoints endpoint WHERE (endpoint.provider = 'slack'
|
||||
AND endpoint.status = 'draft' AND endpoint.setup->>'step' = 'provider_setup'
|
||||
AND (endpoint.setup - ARRAY['step', 'command']) = '{}'::jsonb
|
||||
AND (endpoint.setup->'command' IS NULL OR endpoint.setup->'command' = 'null'::jsonb
|
||||
OR jsonb_typeof(endpoint.setup->'command') = 'string')
|
||||
AND endpoint.activated_at IS NULL AND endpoint.provider_account_id IS NULL
|
||||
AND endpoint.bot_external_id IS NULL AND endpoint.bot_username IS NULL
|
||||
AND endpoint.last_event_at IS NULL AND endpoint.last_publication_at IS NULL
|
||||
AND endpoint.health_message IS NULL AND endpoint.last_error IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id)
|
||||
AND EXISTS (SELECT 1 FROM tool_connections draft_connection
|
||||
WHERE draft_connection.id = endpoint.connection_id AND draft_connection.company_id = endpoint.company_id
|
||||
AND draft_connection.connection_purpose = 'channel' AND draft_connection.transport = 'chat_sdk'
|
||||
AND draft_connection.status = 'draft' AND draft_connection.enabled = false)) IS NOT TRUE`),
|
||||
workCheck("chat_endpoints", incomingWakeWorkQuery("chat_endpoints")!),
|
||||
workCheck("routines", `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`),
|
||||
workCheck("routine_runs", `SELECT 1 FROM routine_runs WHERE status NOT IN ('completed', 'failed', 'skipped', 'cancelled')`),
|
||||
workCheck("plugin_jobs", `SELECT 1 FROM plugin_jobs WHERE status <> 'paused'`),
|
||||
@@ -159,13 +150,7 @@ const WORK_CHECKS = [
|
||||
// Saved outbound MCP configuration and disabled drafts have no independent
|
||||
// listener. Keep local processes, channels and GitHub webhook bindings awake;
|
||||
// accepted calls, credential recovery and delivery work have separate fences.
|
||||
workCheck("tool_connections", `SELECT 1 FROM tool_connections c WHERE c.status <> 'archived' AND (
|
||||
(c.status = 'draft' AND c.enabled = false)
|
||||
OR (c.status = 'active' AND c.connection_purpose = 'ai' AND c.transport = 'runtime_auth')
|
||||
OR (c.status = 'active' AND c.connection_purpose = 'tool' AND c.transport = 'mcp_remote'
|
||||
AND NOT EXISTS (SELECT 1 FROM connection_grants g WHERE g.connection_id = c.id
|
||||
AND (g.company_id <> c.company_id OR g.provider_tenant->'github' IS NOT NULL)))
|
||||
) IS NOT TRUE`),
|
||||
workCheck("tool_connections", incomingWakeWorkQuery("tool_connections")!),
|
||||
// An explicit board escalation has no automatic retry. Retain the visible
|
||||
// action and all execution/cleanup checks while the board decides what to do.
|
||||
workCheck("issue_recovery_actions", `SELECT 1 FROM issue_recovery_actions a WHERE a.status NOT IN ('resolved', 'cancelled')
|
||||
@@ -294,6 +279,8 @@ function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus
|
||||
* Only a caller holding the exact idle owner may receive an empty report.
|
||||
*/
|
||||
export interface IdleSleepSafetyOptions {
|
||||
incomingWake?: VerifiedIdleCapabilities;
|
||||
deadlineCapability?: VerifiedIdleCapabilities;
|
||||
collectDiagnostics?: true;
|
||||
}
|
||||
|
||||
@@ -310,12 +297,15 @@ export async function readIdleSleepSafety(
|
||||
scheduledIssueMonitors = false,
|
||||
options: IdleSleepSafetyOptions = {},
|
||||
): Promise<IdleSleepSafety> {
|
||||
let incomingProof: IdleIncomingWakeProof | undefined;
|
||||
const diagnostic = options.collectDiagnostics ? createIdleSleepDiagnostics() : undefined;
|
||||
const report = (backgroundWork: IdleLocalWork, scheduledWake?: ScheduledWake): IdleSleepSafety => {
|
||||
const diagnostics = diagnostic?.report();
|
||||
if (backgroundWork === "none" && diagnostics?.checks.some(check =>
|
||||
check.state === "unknown" || check.state === "not_checked")) backgroundWork = "unknown";
|
||||
return { version: backgroundWork === "none" && scheduledWake ? 2 : 1, backgroundWork,
|
||||
return { version: backgroundWork === "none" && incomingProof ? 3 : backgroundWork === "none" && scheduledWake ? 2 : 1, backgroundWork,
|
||||
...(backgroundWork === "none" && incomingProof ? { incomingWake: incomingProof.capability,
|
||||
...(incomingProof.apiWakeAdmission ? { apiWakeAdmission: incomingProof.apiWakeAdmission } : {}) } : {}),
|
||||
...(backgroundWork === "none" && scheduledWake ? { scheduledWake } : {}),
|
||||
...(diagnostics ? { diagnostics } : {}) };
|
||||
};
|
||||
@@ -360,9 +350,14 @@ export async function readIdleSleepSafety(
|
||||
let scheduledWake: ScheduledWake | undefined;
|
||||
const blocked = await db.transaction(async (tx) => {
|
||||
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
|
||||
if (options.incomingWake?.incomingWake) {
|
||||
if (!ownerId || ownerId !== before.ownerId) return undefined;
|
||||
incomingProof = await readIdleIncomingWakeProof(tx, { capabilities: options.incomingWake, ownerId, receiverOwner: db });
|
||||
}
|
||||
const checks = WORK_CHECKS.filter(check =>
|
||||
(!scheduledRoutines || check.reason !== "routines")
|
||||
&& (!scheduledIssueMonitors || check.reason !== "issues"));
|
||||
&& (!scheduledIssueMonitors || check.reason !== "issues"))
|
||||
.map(check => ({ ...check, query: incomingWakeWorkQuery(check.reason, incomingProof) ?? check.query }));
|
||||
checks.push(workCheck("agents.company_hold", sql`SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id
|
||||
WHERE ${companyHoldMismatch()}`));
|
||||
checks.push(workCheck("heartbeat_runs.budget_cancellation", sql`SELECT 1 FROM heartbeat_runs WHERE ${pendingBudgetRunCancellationPredicate()}`));
|
||||
@@ -447,6 +442,9 @@ export async function readIdleSleepSafety(
|
||||
diagnostic?.record("inspection.hold", "unknown");
|
||||
return report("present");
|
||||
}
|
||||
// Legacy alarm kinds cannot be silently combined with a new ingress proof.
|
||||
// The negotiated deadline catalog supplies the composed v3 alarm separately.
|
||||
if (incomingProof && scheduledWake) return report("unknown");
|
||||
return report(local, scheduledWake);
|
||||
} catch {
|
||||
// Missing migrations, unknown state, malformed configuration, timeouts and
|
||||
|
||||
Reference in new issue
Block a user