feat(server): bind idle credential eligibility to incoming wake proof (#18639)

Reviewed and verified at d1e3afa4af.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-11-03 14:30:07 -06:00
1 parent 0b87308d5d
commit 5877d751c0
31 files changed
+2071 -53

No files matched your search

+11
View File
@@ -136,6 +136,7 @@ import { notifyChatDeliveryWork, notifyChatPublicationWork } from "./chat-work-n
import { voiceSessionStore } from "./voice/voice-session-store.js";
import { createChatRuntime, type ChatRuntime } from "./chat-runtime.js";
import { registerIdleLocalWorkProbe } from "./idle-local-work.js";
import type { ChatEndpointRuntime } from "./voice/voice-runtime.js";
import { fastResponseService, fastResponseSourceCurrent } from "./fast-responses.js";
@@ -517,6 +518,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
version: string;
}
>();
// SDK initialize() and the final authority read continue after the registry's
// replacement tail settles. Neither that work nor a failed retirement may
// disappear just because the archived endpoint is no longer returned by get().
const unregisterRuntimeIdleProbe = registerIdleLocalWorkProbe(() =>
runtimeInitializations.size > 0 || runtime.hasUnsettledLifecycle() ? "present" : "none");
const { recordCurrentMicrosoftTeamsRoute } = createTeamsRoutes(db, { runtime, runtimeCallbackEndpoint });
const persistence = createChatSdkStatePersistence(db);
const fetchImpl = options.fetch ?? globalThis.fetch;
@@ -3622,12 +3628,17 @@ return {
// Stop every provider runtime before releasing leader rows. This gives a
// standby an immediate takeover path without overlapping Gateway sockets.
await runtime.shutdown();
await Promise.allSettled([...runtimeInitializations.values()].map(initialization => initialization.promise));
await slackRegistration.close();
await Promise.all(
ownedDiscordGateways.map((ownership) =>
releaseDiscordGatewayLeaseRow(ownership),
),
);
if (runtimeInitializations.size > 0 || runtime.hasUnsettledLifecycle()) {
throw new Error("Chat runtime lifecycle is still unsettled after shutdown");
}
unregisterRuntimeIdleProbe();
},
};
}
+5
View File
@@ -23,6 +23,11 @@ export class ChatRuntime {
return [...this.endpoints.values()];
}
/** Retirement failures retain their owner even after get/list hide it. */
hasUnsettledLifecycle(): boolean {
return this.retiringEndpoints.size > 0 || this.lifecycleTails.size > 0;
}
private enqueueLifecycle<T>(
endpointId: string,
operation: () => Promise<T>,
@@ -241,6 +241,8 @@ import {
type CreateChatSdkEndpointRuntimeOptions,
} from "./chat-sdk-runtime.js";
import { createChatRuntime } from "./chat-runtime.js";
const persistence: ChatSdkStatePersistence = {
async compareAndSet() {
return true;
@@ -1152,6 +1154,31 @@ describe("Chat SDK endpoint runtime", () => {
await registry.shutdown();
});
it.each([
{ name: "SDK", create: createChatSdkRuntime },
{ name: "shared", create: createChatRuntime },
])("keeps $name lifecycle work visible while hidden retirement fails, until its owner retries", async ({ create }) => {
const registry = create();
const first = await registry.replaceEndpoint(registryOptions());
expect(registry.hasUnsettledLifecycle()).toBe(false);
const held = deferred();
const shutdown = vi.spyOn(first, "shutdown").mockReturnValueOnce(held.promise).mockResolvedValue(undefined);
const removal = registry.removeEndpoint("endpoint-1");
const result = removal.catch((error: Error) => error.message);
expect(registry.hasUnsettledLifecycle()).toBe(true);
await vi.waitFor(() => expect(shutdown).toHaveBeenCalledOnce());
expect(registry.get("endpoint-1")).toBeNull();
expect(registry.list()).toEqual([]);
expect(registry.hasUnsettledLifecycle()).toBe(true);
held.reject(new Error("retirement uncertain"));
expect(await result).toBe("retirement uncertain");
expect(registry.hasUnsettledLifecycle()).toBe(true);
await registry.removeEndpoint("endpoint-1");
expect(shutdown).toHaveBeenCalledTimes(2);
expect(registry.hasUnsettledLifecycle()).toBe(false);
await registry.shutdown();
});
it("retains failed retirement ownership and only constructs a replacement after an explicit successful retry", async () => {
const registry = createChatSdkRuntime();
const first = await registry.replaceEndpoint(registryOptions());
+5
View File
@@ -3195,6 +3195,11 @@ export class ChatSdkRuntime {
return [...this.endpoints.values()];
}
/** Retirement failures retain their owner even after get/list hide it. */
hasUnsettledLifecycle(): boolean {
return this.retiringEndpoints.size > 0 || this.lifecycleTails.size > 0;
}
private enqueueLifecycle<T>(
endpointId: string,
operation: () => Promise<T>,
@@ -1,3 +1,4 @@
import { parseIdleCapabilities, type IdleCapabilities } from "./idle-incoming-wake-capability.js";
import { createPublicKey, timingSafeEqual, verify, type JsonWebKey } from "node:crypto";
import { eq } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -491,6 +492,7 @@ export type CloudControlClaims = {
sub: string;
action: CloudControlAction;
requestId: string;
idleCapabilities?: IdleCapabilities;
iat: number;
exp: number;
};
@@ -572,6 +574,10 @@ export function verifyCloudControlAssertion(input: {
) {
throw new Error("Cloud control assertion is expired or has an invalid lifetime");
}
if (payload.idleCapabilities !== undefined) {
if (payload.action !== "task-drain:read") throw new Error("Idle capabilities require task-drain:read");
payload.idleCapabilities = parseIdleCapabilities(payload.idleCapabilities);
}
// Consumed LAST, only after every other check passed: a rejected
// assertion must not burn its request id, or an attacker could deny a
// legitimate call by replaying a mangled copy of it first.
@@ -10,3 +10,6 @@ export function wakeConnectionEvents(owner: object): boolean {
wake();
return true;
}
/** Inspection never emits a wake or polls the broker. */
export const hasConnectionEventWakeReceiver = (owner: object): boolean => receivers.has(owner);
@@ -146,7 +146,7 @@ function normalizeLeasedPayload(event: LeasedEvent): Record<string, unknown> {
return compact(base);
}
function bindingRows(rows: Array<{
export function githubConnectionEventBindings(rows: Array<{
grant: typeof connectionGrants.$inferSelect;
connection: typeof toolConnections.$inferSelect;
}>): GitHubBinding[] {
@@ -259,7 +259,7 @@ export function githubConnectionEventService(
eq(toolConnections.enabled, true),
sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status <> 'archived')`,
));
return bindingRows(rows);
return githubConnectionEventBindings(rows);
}
async function applyPullRequestEvent(database: Db, companyId: string, event: LeasedEvent, effects: Array<(committedDb: Db) => Promise<void>>) {
@@ -0,0 +1,27 @@
import { describe, expect, it } from "vitest";
import { IdleApiActivity } from "./idle-api-activity.js";
describe("authenticated API idle activity", () => {
it("is passive without successful authentication and expires without timers or dependency work", () => {
let monotonic = 1;
const activity = new IdleApiActivity(() => monotonic);
expect(activity.active()).toBe(false);
activity.record();
expect(activity.active()).toBe(true);
monotonic += 599_999;
expect(activity.active()).toBe(true);
monotonic++;
expect(activity.active()).toBe(false);
});
it("extends only from the actual latest successful use and ignores wall clock changes", () => {
let monotonic = 1;
const activity = new IdleApiActivity(() => monotonic);
activity.record();
monotonic += 300_000;
activity.record();
monotonic += 300_001;
expect(activity.active()).toBe(true);
monotonic += 299_999;
expect(activity.active()).toBe(false);
});
});
+12
View File
@@ -0,0 +1,12 @@
/** Authenticated API use is real activity; rejecting an unknown key is not. */
export class IdleApiActivity {
private until = 0;
constructor(private readonly now: () => number = () => performance.now()) {}
record(): void {
this.until = Math.max(this.until, this.now() + 10 * 60_000);
}
active(): boolean {
return this.now() < this.until;
}
}
export const idleApiActivity = new IdleApiActivity();
@@ -0,0 +1,33 @@
import { createHash } from "node:crypto";
import { describe, expect, it } from "vitest";
import { apiWakeFingerprint, createApiWakeAdmission, parseApiWakeAdmission, matchesApiWakeAdmission } from "./idle-api-wake-admission.js";
const token = "api-wake-vector-token";
const keyHash = createHash("sha256").update(token).digest("hex");
const input = { stackId: "vector-stack", ownerId: "00000000-0000-4000-8000-000000000001", capabilityGeneration: "a".repeat(64), challenge: "c".repeat(64) };
const fingerprint = apiWakeFingerprint({ ...input, keyHash });
const receipt = createApiWakeAdmission({ ...input, keys: [{ fingerprint, expiresAt: null }] });
describe("API wake receipt wire vectors", () => {
it("uses exact domain-separated vectors and exports neither token nor database hash", () => {
expect(fingerprint).toBe("6478041b90f4665eeca8403186fefedc456bf1322f38dab9fb89212eecaafd43");
expect(JSON.stringify(receipt)).not.toContain(token);
expect(JSON.stringify(receipt)).not.toContain(keyHash);
expect(matchesApiWakeAdmission(receipt, { stackId: input.stackId, token, now: new Date("2099-01-01").getTime() })).toBe(true);
expect(matchesApiWakeAdmission(receipt, { stackId: "other", token, now: 0 })).toBe(false);
expect(matchesApiWakeAdmission(receipt, { stackId: input.stackId, token: "unknown", now: 0 })).toBe(false);
});
it("enforces board expiry and canonical unique bounded complete receipts", () => {
const changedChallenge = createApiWakeAdmission({ ...input, challenge: "b".repeat(64), keys: receipt.keys });
expect(matchesApiWakeAdmission(changedChallenge, { stackId: input.stackId, token, now: 0 })).toBe(false);
const expiring = createApiWakeAdmission({ ...input, keys: [{ fingerprint, expiresAt: new Date(1000).toISOString() }] });
expect(matchesApiWakeAdmission(expiring, { stackId: input.stackId, token, now: 999 })).toBe(true);
expect(matchesApiWakeAdmission(expiring, { stackId: input.stackId, token, now: 1000 })).toBe(false);
for (const invalid of [
{ ...receipt, generation: "b".repeat(64) }, { ...receipt, challenge: "b".repeat(64) },
{ ...receipt, extra: true }, { ...receipt, keys: [...receipt.keys, ...receipt.keys] },
{ ...receipt, keys: Array.from({ length: 1001 }, () => receipt.keys[0]) },
{ ...receipt, keys: [{ fingerprint, expiresAt: "2099-01-01" }] },
{ ...receipt, keys: [{ fingerprint: "d".repeat(64), expiresAt: null }, { fingerprint: "a".repeat(64), expiresAt: null }] },
]) expect(() => parseApiWakeAdmission(invalid)).toThrow();
});
});
@@ -0,0 +1,56 @@
import { createHash } from "node:crypto";
export type ApiWakeClaim = { generation: string; fingerprint: string };
export interface ApiWakeAdmission {
version: 1;
stackId: string;
ownerId: string;
capabilityGeneration: string;
challenge: string;
generation: string;
keys: { fingerprint: string; expiresAt: string | null }[];
}
const digest = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
const object = (value: unknown): value is Record<string, unknown> => Boolean(value && typeof value === "object" && !Array.isArray(value));
const exact = (value: Record<string, unknown>, keys: string[]) => Object.keys(value).length === keys.length && Object.keys(value).every(key => keys.includes(key));
const generation = (value: Omit<ApiWakeAdmission, "generation">) => createHash("sha256").update(JSON.stringify([
"paperclip-api-wake-receipt-v1", value.version, value.stackId, value.ownerId, value.capabilityGeneration, value.challenge,
value.keys.map(key => [key.fingerprint, key.expiresAt]),
])).digest("hex");
export function apiWakeFingerprint(input: { stackId: string; ownerId: string; challenge: string; keyHash: string }): string {
if (!digest(input.keyHash) || !digest(input.challenge)) throw new Error("Invalid API wake fingerprint input");
return createHash("sha256").update(JSON.stringify(["paperclip-api-wake-v1", input.stackId, input.ownerId, input.challenge, input.keyHash])).digest("hex");
}
/** Strict bounded canonical wire contract. Parsing never grants tenant permissions. */
export function parseApiWakeAdmission(value: unknown): ApiWakeAdmission {
if (!object(value) || !exact(value, ["version", "stackId", "ownerId", "capabilityGeneration", "challenge", "generation", "keys"])
|| value.version !== 1 || typeof value.stackId !== "string" || !/^[A-Za-z0-9_-]{1,200}$/.test(value.stackId)
|| typeof value.ownerId !== "string" || !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value.ownerId)
|| !digest(value.capabilityGeneration) || !digest(value.challenge) || !digest(value.generation)
|| !Array.isArray(value.keys) || value.keys.length > 1000) throw new Error("Invalid API wake receipt");
let previous = "";
for (const key of value.keys) {
if (!object(key) || !exact(key, ["fingerprint", "expiresAt"]) || !digest(key.fingerprint) || key.fingerprint <= previous
|| !(key.expiresAt === null || typeof key.expiresAt === "string" && key.expiresAt.length <= 30 && Number.isFinite(Date.parse(key.expiresAt))
&& new Date(key.expiresAt).toISOString() === key.expiresAt)) throw new Error("Invalid API wake receipt key");
previous = key.fingerprint;
}
const parsed = value as unknown as ApiWakeAdmission;
if (generation(parsed) !== parsed.generation) throw new Error("API wake receipt generation changed");
return { version: 1, stackId: parsed.stackId, ownerId: parsed.ownerId, capabilityGeneration: parsed.capabilityGeneration,
challenge: parsed.challenge, generation: parsed.generation,
keys: parsed.keys.map(key => ({ fingerprint: key.fingerprint, expiresAt: key.expiresAt })) };
}
export function createApiWakeAdmission(input: Omit<ApiWakeAdmission, "version" | "generation">): ApiWakeAdmission {
const canonical = { version: 1 as const, ...input, keys: [...input.keys].sort((a, b) => a.fingerprint.localeCompare(b.fingerprint)) };
return parseApiWakeAdmission({ ...canonical, generation: generation(canonical) });
}
export function authorizeApiWakeAdmission(receipt: ApiWakeAdmission, input: { stackId: string; token: string; now: number }): ApiWakeClaim | undefined {
if (receipt.stackId !== input.stackId || input.token.length < 1 || input.token.length > 8192 || /\s/.test(input.token)) return;
const fingerprint = apiWakeFingerprint({ ...receipt, keyHash: createHash("sha256").update(input.token).digest("hex") });
return receipt.keys.some(key => key.fingerprint === fingerprint && (key.expiresAt === null || Date.parse(key.expiresAt) > input.now))
? { generation: receipt.generation, fingerprint } : undefined;
}
export function matchesApiWakeAdmission(receipt: ApiWakeAdmission, input: { stackId: string; token: string; now: number }): boolean {
return authorizeApiWakeAdmission(receipt, input) !== undefined;
}
@@ -0,0 +1,113 @@
import { generateKeyPairSync, sign } from "node:crypto";
import { describe, expect, it } from "vitest";
import { parseIdleCapabilities } from "./idle-incoming-wake-capability.js";
import {
verifyCloudControlAssertion,
resetCloudControlReplayFenceForTests,
CLOUD_CONTROL_AUDIENCE,
CLOUD_CONTROL_JWS_TYPE,
CLOUD_RUNTIME_IDENTITY_ISSUER,
} from "./cloud-runtime-identity.js";
const incoming = {
version: 1 as const,
ownerId: "00000000-0000-4000-8000-000000000001",
generation: "a".repeat(64),
apiKeyProof: { version: 1 as const, challenge: "c".repeat(64) },
routes: ["tenant-api-http-v2" as const],
};
const invalid: unknown[] = [
null,
{},
{ other: true },
{ deadlineCatalog: "other" },
{ incomingWake: { ...incoming, version: 2 } },
{ incomingWake: { ...incoming, routes: ["all"] } },
{
incomingWake: {
...incoming,
routes: ["tenant-api-http-v2", "tenant-api-http-v2"],
},
},
{ incomingWake: { ...incoming, ownerId: "other" } },
{ incomingWake: { ...incoming, extra: true } },
{
incomingWake: {
...incoming,
githubBroker: {
instanceId: "instance",
bindingsFingerprint: "b".repeat(64),
},
},
},
{ incomingWake: { ...incoming, routes: ["github-broker-events-v1"] } },
];
describe("signed idle capabilities", () => {
it.each(invalid)("rejects unknown/partial capabilities: %j", (value) => {
expect(() => parseIdleCapabilities(value)).toThrow();
});
it("detaches and freezes the exact known catalogs", () => {
const source = {
incomingWake: { ...incoming, routes: [...incoming.routes] },
deadlineCatalog: "domain_deadlines_v1",
};
const parsed = parseIdleCapabilities(source);
source.incomingWake.routes.length = 0;
expect(parsed.incomingWake?.routes).toEqual(["tenant-api-http-v2"]);
expect(Object.isFrozen(parsed.incomingWake?.routes)).toBe(true);
});
it("binds capability to signed stack, action and replay fence", () => {
resetCloudControlReplayFenceForTests();
const pair = generateKeyPairSync("ed25519");
const now = new Date("2099-01-01T00:00:00Z");
const jwk = {
...pair.publicKey.export({ format: "jwk" }),
kid: "idle",
use: "sig",
alg: "EdDSA",
};
const env = {
PAPERCLIP_CLOUD_STACK_ID: "stack",
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [jwk] }),
};
const encode = (v: unknown) =>
Buffer.from(JSON.stringify(v)).toString("base64url");
const token = (overrides: object = {}) => {
const header = encode({
alg: "EdDSA",
typ: CLOUD_CONTROL_JWS_TYPE,
kid: "idle",
});
const body = encode({
v: 1,
iss: CLOUD_RUNTIME_IDENTITY_ISSUER,
aud: CLOUD_CONTROL_AUDIENCE,
sub: "stack",
action: "task-drain:read",
requestId: "same",
iat: now.getTime() / 1000,
exp: now.getTime() / 1000 + 60,
idleCapabilities: {
incomingWake: incoming,
deadlineCatalog: "domain_deadlines_v1",
},
...overrides,
});
return `${header}.${body}.${sign(null, Buffer.from(`${header}.${body}`), pair.privateKey).toString("base64url")}`;
};
const verify = (compactJws: string) =>
verifyCloudControlAssertion({
compactJws,
env,
now,
expectedAction: "task-drain:read",
});
expect(() => verify(token({ sub: "foreign" }))).toThrow();
expect(() => verify(token({ action: "task-drain:start" }))).toThrow();
expect(() =>
verify(token({ idleCapabilities: { deadlineCatalog: "unknown" } })),
).toThrow();
expect(verify(token()).idleCapabilities?.incomingWake).toEqual(incoming);
expect(() => verify(token())).toThrow(/already been used/);
});
});
@@ -0,0 +1,93 @@
/** Closed Cloud-owned ingress catalog. No credential material belongs in this proof. */
export interface IdleIncomingWakeCapability {
version: 1;
ownerId: string;
generation: string;
routes: ("tenant-api-http-v2" | "github-broker-events-v1")[];
apiKeyProof?: { version: 1; challenge: string };
githubBroker?: { instanceId: string; bindingsFingerprint: string };
}
export interface IdleCapabilities {
incomingWake?: IdleIncomingWakeCapability;
deadlineCatalog?: "domain_deadlines_v1";
}
const object = (v: unknown): v is Record<string, unknown> =>
Boolean(v && typeof v === "object" && !Array.isArray(v));
const exact = (v: Record<string, unknown>, keys: string[]) =>
Object.keys(v).every((k) => keys.includes(k));
const digest = (v: unknown): v is string =>
typeof v === "string" && /^[a-f0-9]{64}$/.test(v);
export function parseIdleCapabilities(value: unknown): IdleCapabilities {
if (
!object(value) ||
!exact(value, ["incomingWake", "deadlineCatalog"]) ||
!Object.keys(value).length
)
throw new Error("Invalid idle capabilities");
if (
value.deadlineCatalog !== undefined &&
value.deadlineCatalog !== "domain_deadlines_v1"
)
throw new Error("Unknown idle deadline catalog");
if (value.incomingWake !== undefined) {
const v = value.incomingWake;
if (
!object(v) ||
!exact(v, [
"version",
"ownerId",
"generation",
"routes",
"githubBroker",
"apiKeyProof",
]) ||
v.version !== 1 ||
typeof v.ownerId !== "string" ||
!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(
v.ownerId,
) ||
!digest(v.generation) ||
!Array.isArray(v.routes) ||
v.routes.length < 1 ||
v.routes.length > 2 ||
v.routes.some(
(r) => !["tenant-api-http-v2", "github-broker-events-v1"].includes(r),
) ||
new Set(v.routes).size !== v.routes.length ||
[...v.routes].sort().join() !== v.routes.join()
)
throw new Error("Invalid incoming wake capability");
if (v.routes.includes("tenant-api-http-v2")) {
const proof = v.apiKeyProof;
if (!object(proof) || Object.keys(proof).length !== 2 || !exact(proof, ["version", "challenge"])
|| proof.version !== 1 || !digest(proof.challenge)) throw new Error("Invalid API wake challenge");
} else if (v.apiKeyProof !== undefined) throw new Error("Unexpected API wake challenge");
if (v.routes.includes("github-broker-events-v1")) {
const b = v.githubBroker;
if (
!object(b) ||
!exact(b, ["instanceId", "bindingsFingerprint"]) ||
typeof b.instanceId !== "string" ||
!/^[A-Za-z0-9_-]{1,200}$/.test(b.instanceId) ||
!digest(b.bindingsFingerprint)
)
throw new Error("Invalid GitHub wake binding");
} else if (v.githubBroker !== undefined)
throw new Error("Unexpected GitHub wake binding");
}
// Return a detached immutable snapshot; request callers cannot mutate a verified claim.
const parsed = JSON.parse(JSON.stringify(value)) as IdleCapabilities;
if (parsed.incomingWake) {
Object.freeze(parsed.incomingWake.routes);
if (parsed.incomingWake.apiKeyProof) Object.freeze(parsed.incomingWake.apiKeyProof);
if (parsed.incomingWake.githubBroker)
Object.freeze(parsed.incomingWake.githubBroker);
Object.freeze(parsed.incomingWake);
}
return Object.freeze(parsed);
}
declare const verified: unique symbol;
export type VerifiedIdleCapabilities = IdleCapabilities & {
readonly [verified]: true;
};
@@ -0,0 +1,184 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import {
readIdleIncomingWakeProof,
incomingWakeFingerprint,
} from "./idle-incoming-wake.js";
import { registerConnectionEventWake } from "./connection-event-wake.js";
import type { VerifiedIdleCapabilities } from "./idle-incoming-wake-capability.js";
const ownerId = "00000000-0000-4000-8000-000000000001";
function fixture(count: number) {
const endpoints = Array.from({ length: count }, (_, i) => ({
endpoint: {
id: `endpoint-${i}`,
companyId: "company",
status: "active",
activatedAt: new Date(0),
botExternalId: `bot-${i}`,
providerAccountId: `app-${i}`,
setup: {
step: "complete",
github: {
cloudRegistrationId: String(i).padStart(43, "a"),
registrationStatus: "completed",
},
},
},
connection: { id: `connection-${i}` },
}));
const registrations = endpoints.map(({ endpoint }) => ({
endpointId: endpoint.id,
companyId: endpoint.companyId,
userId: "owner",
cloudId: endpoint.setup.github.cloudRegistrationId,
}));
const tuples = registrations.map((row) => [
`github-app:${row.cloudId}`,
row.companyId,
row.userId,
"github.bot",
`gateway:${row.cloudId}`,
row.cloudId,
row.cloudId,
]);
const capability = {
incomingWake: {
version: 1,
ownerId,
generation: "a".repeat(64),
routes: ["github-broker-events-v1"],
githubBroker: {
instanceId: "fixture-instance",
bindingsFingerprint: incomingWakeFingerprint(tuples),
},
},
} as VerifiedIdleCapabilities;
return { endpoints, registrations, capability };
}
/** Named DB boundary for cardinality/elapsed guards; PostgreSQL fixture covers real SQL separately. */
function database(replies: Array<() => unknown[]>) {
let index = 0;
const select = vi.fn(() => {
const reply = replies[index++];
const query = {
from: () => query,
innerJoin: () => query,
where: () => query,
limit: async () => reply?.() ?? [],
};
return query;
});
const execute = vi.fn(async () => []);
return {
db: { select, execute } as unknown as Pick<Db, "select" | "execute">,
select,
execute,
};
}
const stops: Array<() => void> = [];
function input(
db: object,
capabilities: VerifiedIdleCapabilities,
now?: () => number,
) {
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "fixture-instance");
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", "fixture");
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", "fixture");
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "http://127.0.0.1");
stops.push(registerConnectionEventWake(db, () => {}));
return { ownerId, receiverOwner: db, capabilities, now };
}
afterEach(() => {
for (const stop of stops.splice(0)) stop();
vi.unstubAllEnvs();
});
describe("bounded incoming wake owner projection", () => {
it("uses three bounded selects for a thousand bots and retains exact per-bot authority", async () => {
const f = fixture(1000),
sql = database([() => [], () => f.endpoints, () => f.registrations]);
const result = await readIdleIncomingWakeProof(
sql.db,
input(sql.db, f.capability),
);
expect(sql.select).toHaveBeenCalledTimes(3);
expect(sql.execute).toHaveBeenCalledTimes(4);
expect(result?.endpointIds).toHaveLength(1000);
expect(result?.connectionIds).toHaveLength(1000);
});
it.each(["duplicate", "foreign", "missing"])(
"rejects %s registration authority",
async (fault) => {
const f = fixture(1);
let registrations = f.registrations;
if (fault === "duplicate")
registrations = [...registrations, ...registrations];
if (fault === "foreign")
registrations = registrations.map((r) => ({
...r,
companyId: "foreign",
}));
if (fault === "missing") registrations = [];
const sql = database([() => [], () => f.endpoints, () => registrations]);
await expect(
readIdleIncomingWakeProof(sql.db, input(sql.db, f.capability)),
).rejects.toThrow("registration authority");
},
);
it("stops the complete inspection when its monotonic budget expires before another query", async () => {
const f = fixture(1);
let now = 0;
const sql = database([
() => {
now = 2001;
return [];
},
() => f.endpoints,
() => f.registrations,
]);
await expect(
readIdleIncomingWakeProof(
sql.db,
input(sql.db, f.capability, () => now),
),
).rejects.toThrow("budget exhausted");
expect(sql.select).toHaveBeenCalledTimes(1);
expect(sql.execute).toHaveBeenCalledTimes(1);
});
it("fails closed at row limits before expanding work or issuing registration queries", async () => {
const f = fixture(1001),
sql = database([() => [], () => f.endpoints, () => f.registrations]);
await expect(
readIdleIncomingWakeProof(sql.db, input(sql.db, f.capability)),
).rejects.toThrow("Too many GitHub endpoints");
expect(sql.select).toHaveBeenCalledTimes(2);
});
});
describe("bounded API admission projection", () => {
const api = { incomingWake: { version: 1, ownerId, generation: "a".repeat(64), routes: ["tenant-api-http-v2"],
apiKeyProof: { version: 1, challenge: "c".repeat(64) } } } as VerifiedIdleCapabilities;
it.each(["overflow", "duplicate", "malformed"])("refuses complete eligibility for %s key state", async kind => {
vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack");
const row = { keyHash: "a".repeat(64), expiresAt: null };
const rows = kind === "overflow" ? Array.from({ length: 1001 }, () => row)
: kind === "duplicate" ? [row, row] : [{ keyHash: "not-a-digest", expiresAt: null }];
const execute = vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce(rows);
await expect(readIdleIncomingWakeProof({ execute } as unknown as Pick<Db, "select" | "execute">,
{ ownerId, receiverOwner: {}, capabilities: api })).rejects.toThrow();
expect(execute).toHaveBeenCalledTimes(2);
});
it("has an explicit complete empty-key receipt and shares the inspection time budget", async () => {
vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack");
const execute = vi.fn(async () => []);
const proof = await readIdleIncomingWakeProof({ execute } as unknown as Pick<Db, "select" | "execute">,
{ ownerId, receiverOwner: {}, capabilities: api });
expect(proof?.apiWakeAdmission?.keys).toEqual([]);
let clock = 0;
const slow = vi.fn(async () => { clock += 1100; return []; });
await expect(readIdleIncomingWakeProof({ execute: slow } as unknown as Pick<Db, "select" | "execute">,
{ ownerId, receiverOwner: {}, capabilities: api, now: () => clock })).rejects.toThrow(/budget exhausted/);
});
});
+359
View File
@@ -0,0 +1,359 @@
import { apiWakeFingerprint, createApiWakeAdmission, type ApiWakeAdmission } from "./idle-api-wake-admission.js";
import { createHash } from "node:crypto";
import { and, eq, inArray, sql, type SQL } from "drizzle-orm";
import {
chatEndpoints,
chatGitHubRegistrations,
connectionGrants,
toolConnections,
type Db,
} from "@paperclipai/db";
import type {
IdleIncomingWakeCapability,
VerifiedIdleCapabilities,
} from "./idle-incoming-wake-capability.js";
import { githubConnectionEventBindings } from "./github-connection-events.js";
import { paperclipCloudConnectorConfigFromEnv } from "./paperclip-cloud-connector.js";
import { hasConnectionEventWakeReceiver } from "./connection-event-wake.js";
export interface IdleIncomingWakeProof {
capability: IdleIncomingWakeCapability;
apiWakeAdmission?: ApiWakeAdmission;
connectionIds: string[];
endpointIds: string[];
/** A broker enrollment cannot discharge these credential continuity obligations. */
requiredContinuityConnectionIds: string[];
}
export const incomingWakeFingerprint = (bindings: string[][]) =>
createHash("sha256")
.update(
JSON.stringify(
bindings
.map((b) => [...b])
.sort((a, b) =>
JSON.stringify(a) < JSON.stringify(b)
? -1
: JSON.stringify(a) > JSON.stringify(b)
? 1
: 0,
),
),
)
.digest("hex");
/** Read current binding authority in the safety transaction. No network call, renewal or write. */
export async function readIdleIncomingWakeProof(
tx: Pick<Db, "select" | "execute">,
input: {
capabilities?: VerifiedIdleCapabilities;
ownerId: string;
receiverOwner: object;
now?: () => number;
},
): Promise<IdleIncomingWakeProof | undefined> {
const capability = input.capabilities?.incomingWake;
if (!capability) return;
if (capability.ownerId !== input.ownerId)
throw new Error("Incoming wake owner changed");
const proof: IdleIncomingWakeProof = {
capability,
connectionIds: [],
endpointIds: [],
requiredContinuityConnectionIds: [],
};
const monotonic = input.now ?? (() => performance.now());
const started = monotonic();
const remaining = () => {
const ms = 2_000 - (monotonic() - started);
if (!Number.isFinite(ms) || ms <= 0)
throw new Error("Incoming wake inspection budget exhausted");
return Math.max(1, Math.floor(ms));
};
const beforeRead = async () => {
// One shared owner budget, not five seconds multiplied by the row count.
await tx.execute(
sql`SELECT set_config('statement_timeout', ${`${remaining()}ms`}, true)`,
);
remaining();
};
if (capability.routes.includes("tenant-api-http-v2")) {
const stackId = process.env.PAPERCLIP_CLOUD_STACK_ID;
if (!stackId || !capability.apiKeyProof) throw new Error("API wake challenge is missing");
await beforeRead();
const rows = await tx.execute<{ keyHash: string; expiresAt: Date | null }>(sql`
SELECT key_hash AS "keyHash", NULL::timestamptz AS "expiresAt" FROM agent_api_keys WHERE revoked_at IS NULL
UNION ALL SELECT key_hash AS "keyHash", expires_at AS "expiresAt" FROM board_api_keys
WHERE revoked_at IS NULL AND (expires_at IS NULL OR expires_at > now()) LIMIT 1001`);
remaining();
if (rows.length > 1000) throw new Error("Too many API keys to certify wake admission");
proof.apiWakeAdmission = createApiWakeAdmission({ stackId, ownerId: input.ownerId,
capabilityGeneration: capability.generation, challenge: capability.apiKeyProof.challenge,
keys: rows.map(row => ({ fingerprint: apiWakeFingerprint({ stackId, ownerId: input.ownerId,
challenge: capability.apiKeyProof!.challenge, keyHash: row.keyHash }),
expiresAt: row.expiresAt === null ? null : new Date(row.expiresAt).toISOString() })),
});
}
if (!capability.routes.includes("github-broker-events-v1")) {
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
remaining();
return proof;
}
const broker = capability.githubBroker;
const config = paperclipCloudConnectorConfigFromEnv();
if (
!broker ||
config?.instanceId !== broker.instanceId ||
!hasConnectionEventWakeReceiver(input.receiverOwner)
)
throw new Error("Incoming GitHub receiver is not bound");
await beforeRead();
const rows = await tx
.select({ grant: connectionGrants, connection: toolConnections })
.from(connectionGrants)
.innerJoin(
toolConnections,
and(
eq(toolConnections.id, connectionGrants.connectionId),
eq(toolConnections.companyId, connectionGrants.companyId),
),
)
.where(
and(
eq(connectionGrants.status, "active"),
eq(toolConnections.status, "active"),
eq(toolConnections.enabled, true),
sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`,
sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`,
sql`exists (select 1 from companies where id=${connectionGrants.companyId} and status <> 'archived')`,
),
)
.limit(1001);
if (rows.length > 1000)
throw new Error("Too many GitHub bindings to inspect");
remaining();
let bindingCount = 0;
for (const { grant, connection } of rows) {
const github = grant.providerTenant?.github;
if (
!github ||
!Array.isArray(github.installationIds) ||
!github.installationIds.length ||
github.installationIds.length > 1000 ||
github.bindingSync ||
github.continuityRetry ||
github.installationIds.some(
(id) => typeof id !== "string" || !/^[0-9]+$/.test(id),
)
)
throw new Error("Unsettled GitHub authority");
bindingCount += github.installationIds.length;
if (bindingCount > 1000)
throw new Error("Too many GitHub bindings to inspect");
proof.requiredContinuityConnectionIds.push(connection.id);
}
const bindings = githubConnectionEventBindings(rows);
if (
bindings.length !==
rows.reduce(
(n, r) => n + r.grant.providerTenant!.github!.installationIds.length,
0,
)
)
throw new Error("Incomplete GitHub binding");
const tuples = bindings.map((b) => [
b.id,
b.companyId,
b.subject,
"github.code",
b.installationId,
b.connectionId,
b.grantId,
]);
await beforeRead();
const endpoints = await tx
.select({ endpoint: chatEndpoints, connection: toolConnections })
.from(chatEndpoints)
.innerJoin(
toolConnections,
and(
eq(toolConnections.id, chatEndpoints.connectionId),
eq(toolConnections.companyId, chatEndpoints.companyId),
),
)
.where(
and(
eq(chatEndpoints.provider, "github"),
eq(chatEndpoints.status, "active"),
eq(toolConnections.status, "active"),
eq(toolConnections.enabled, true),
eq(toolConnections.transport, "chat_sdk"),
eq(toolConnections.connectionPurpose, "channel"),
sql`exists (select 1 from companies where id=${chatEndpoints.companyId} and status <> 'archived')`,
),
)
.limit(1001);
if (endpoints.length > 1000)
throw new Error("Too many GitHub endpoints to inspect");
remaining();
const brokerEndpointIds = endpoints
.filter(
({ endpoint }) =>
typeof endpoint.setup.github?.cloudRegistrationId === "string",
)
.map(({ endpoint }) => endpoint.id);
const registrationUsers = new Map<string, string[]>();
if (brokerEndpointIds.length) {
await beforeRead();
const registrations = await tx
.select({
endpointId: chatGitHubRegistrations.endpointId,
companyId: chatGitHubRegistrations.companyId,
userId: chatGitHubRegistrations.userId,
cloudId: sql<string>`${chatGitHubRegistrations.handoff}->>'cloudId'`,
})
.from(chatGitHubRegistrations)
.where(
and(
inArray(chatGitHubRegistrations.endpointId, brokerEndpointIds),
eq(chatGitHubRegistrations.status, "completed"),
),
)
.limit(10001);
remaining();
if (registrations.length > 10000)
throw new Error("Too many GitHub registrations to inspect");
for (const registration of registrations) {
const key = JSON.stringify([
registration.endpointId,
registration.companyId,
registration.cloudId,
]);
const users = registrationUsers.get(key) ?? [];
users.push(registration.userId);
registrationUsers.set(key, users);
}
}
for (const { endpoint, connection } of endpoints) {
const registration = endpoint.setup.github?.cloudRegistrationId;
if (!registration || !/^[A-Za-z0-9_-]{43}$/.test(registration)) continue;
if (
endpoint.setup.step !== "complete" ||
endpoint.setup.github?.registrationStatus !== "completed" ||
endpoint.setup.github.initialSetupPending ||
endpoint.setup.github.initialRepositoryImportPending ||
!endpoint.activatedAt ||
!endpoint.botExternalId ||
!endpoint.providerAccountId
)
throw new Error("GitHub bot setup has unfinished work");
const users = registrationUsers.get(
JSON.stringify([endpoint.id, endpoint.companyId, registration]),
);
if (users?.length !== 1 || !users[0])
throw new Error("GitHub registration authority changed");
tuples.push([
`github-app:${registration}`,
endpoint.companyId,
users[0],
"github.bot",
`gateway:${registration}`,
registration,
registration,
]);
proof.connectionIds.push(connection.id);
proof.endpointIds.push(endpoint.id);
}
remaining();
if (tuples.length > 1000)
throw new Error("Too many GitHub bindings to inspect");
if (incomingWakeFingerprint(tuples) !== broker.bindingsFingerprint)
throw new Error("GitHub wake bindings changed");
remaining();
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
remaining();
return proof;
}
/** Exact domain predicates selected by stable inventory reason, never SQL-text identity. */
export function incomingWakeWorkQuery(
reason: string,
proof?: IdleIncomingWakeProof,
continuityConnectionIds: readonly string[] = [],
): SQL | undefined {
const connectionIds = [
...(proof?.connectionIds ?? []),
...(proof?.requiredContinuityConnectionIds.filter((id) =>
continuityConnectionIds.includes(id),
) ?? []),
];
if (reason === "agent_api_keys")
return proof?.apiWakeAdmission !== undefined
? sql`SELECT 1 WHERE false`
: sql`SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`;
if (reason === "board_api_keys")
return proof?.apiWakeAdmission !== undefined
? sql`SELECT 1 WHERE false`
: sql`SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL
AND (expires_at IS NULL OR expires_at > now())`;
if (reason === "chat_endpoints") {
const ids = proof?.endpointIds ?? [];
// Only the supported GitHub webhook path gains a removed-history proof.
// Archival commits before cleanup, so status alone cannot discharge work.
return sql`SELECT 1 FROM chat_endpoints endpoint WHERE (((endpoint.provider = 'github'
AND endpoint.status = 'archived' AND endpoint.archived_at IS NOT NULL
AND jsonb_typeof(endpoint.setup) = 'object'
AND CASE WHEN jsonb_typeof(endpoint.setup->'runtimeGeneration') = 'number'
THEN (endpoint.setup->>'runtimeGeneration')::numeric >= 1
AND trunc((endpoint.setup->>'runtimeGeneration')::numeric) = (endpoint.setup->>'runtimeGeneration')::numeric
ELSE false END
AND EXISTS (SELECT 1 FROM tool_connections removed_connection
WHERE removed_connection.id = endpoint.connection_id AND removed_connection.company_id = endpoint.company_id
AND removed_connection.connection_purpose = 'channel' AND removed_connection.transport = 'chat_sdk'
AND removed_connection.status = 'archived' AND removed_connection.enabled = false
AND removed_connection.credential_secret_refs = '[]'::jsonb)
AND NOT EXISTS (SELECT 1 FROM chat_endpoint_leases lease WHERE lease.endpoint_id = endpoint.id)
AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id)
AND NOT EXISTS (SELECT 1 FROM chat_github_registrations registration WHERE registration.endpoint_id = endpoint.id
AND (registration.company_id = endpoint.company_id AND registration.status = 'completed'
AND registration.consumed_at IS NOT NULL) IS NOT TRUE))
OR (endpoint.provider = 'slack'
AND endpoint.status = 'draft' AND endpoint.setup->>'step' = 'provider_setup'
AND (endpoint.setup - ARRAY['step', 'command']) = '{}'::jsonb
AND (endpoint.setup->'command' IS NULL OR endpoint.setup->'command' = 'null'::jsonb
OR jsonb_typeof(endpoint.setup->'command') = 'string')
AND endpoint.activated_at IS NULL AND endpoint.provider_account_id IS NULL
AND endpoint.bot_external_id IS NULL AND endpoint.bot_username IS NULL
AND endpoint.last_event_at IS NULL AND endpoint.last_publication_at IS NULL
AND endpoint.health_message IS NULL AND endpoint.last_error IS NULL
AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id)
AND EXISTS (SELECT 1 FROM tool_connections draft_connection
WHERE draft_connection.id = endpoint.connection_id AND draft_connection.company_id = endpoint.company_id
AND draft_connection.connection_purpose = 'channel' AND draft_connection.transport = 'chat_sdk'
AND draft_connection.status = 'draft' AND draft_connection.enabled = false))) IS NOT TRUE) AND ${
ids.length
? sql`endpoint.id NOT IN (${sql.join(
ids.map((id) => sql`${id}::uuid`),
sql`, `,
)})`
: sql`true`
}`;
}
if (reason === "tool_connections") {
const ids = connectionIds;
return sql`SELECT 1 FROM tool_connections c WHERE (c.status <> 'archived' AND (
(c.status = 'draft' AND c.enabled = false)
OR (c.status = 'active' AND c.connection_purpose = 'ai' AND c.transport = 'runtime_auth')
OR (c.status = 'active' AND c.connection_purpose = 'tool' AND c.transport = 'mcp_remote'
AND NOT EXISTS (SELECT 1 FROM connection_grants g WHERE g.connection_id = c.id
AND (g.company_id <> c.company_id OR g.provider_tenant->'github' IS NOT NULL)))
) IS NOT TRUE) AND ${
ids.length
? sql`c.id NOT IN (${sql.join(
ids.map((id) => sql`${id}::uuid`),
sql`, `,
)})`
: sql`true`
}`;
}
return undefined;
}
+2 -1
View File
@@ -1,3 +1,4 @@
import { idleApiActivity } from "./idle-api-activity.js";
import { promises as fs } from "node:fs";
import path from "node:path";
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
@@ -83,7 +84,7 @@ export async function readIdleLocalWork(options: { backupCheckpoint?: boolean; d
}
let probes: IdleLocalWork;
try {
const local = [...localWorkProbes].map(probe => probe());
const local = [idleApiActivity.active() ? "present" : "none", ...[...localWorkProbes].map(probe => probe())];
probes = local.some(state => state !== "none" && state !== "present") ? "unknown"
: local.includes("present") ? "present" : "none";
} catch { probes = "unknown"; }
+25 -27
View File
@@ -1,3 +1,6 @@
import type { ApiWakeAdmission } from "./idle-api-wake-admission.js";
import { readIdleIncomingWakeProof, incomingWakeWorkQuery, type IdleIncomingWakeProof } from "./idle-incoming-wake.js";
import type { VerifiedIdleCapabilities, IdleIncomingWakeCapability } from "./idle-incoming-wake-capability.js";
import { idleWorkSnapshot } from "./task-admission.js";
import { readIdleLocalWork, type IdleLocalWork } from "./idle-local-work.js";
import { readIdleRoutineSchedule, type ScheduledWake } from "./idle-scheduled-work.js";
@@ -38,10 +41,12 @@ export interface IdleSleepDrainStatus {
}
export interface IdleSleepSafety {
version: 1 | 2;
version: 1 | 2 | 3;
backgroundWork: "none" | "present" | "unknown";
scheduledWake?: ScheduledWake;
diagnostics?: IdleSleepDiagnostics;
incomingWake?: IdleIncomingWakeCapability;
apiWakeAdmission?: ApiWakeAdmission;
}
export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) => Promise<{
@@ -84,9 +89,8 @@ const WORK_CHECKS = [
workCheck("prompt_injection_invocations", `SELECT 1 FROM prompt_injection_invocations WHERE status = 'running'`),
// Persistent machine credentials can receive work without a human opening
// the instance. They need an ingress wake contract before we can sleep.
workCheck("agent_api_keys", `SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`),
workCheck("board_api_keys", `SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL
AND (expires_at IS NULL OR expires_at > now())`),
workCheck("agent_api_keys", incomingWakeWorkQuery("agent_api_keys")!),
workCheck("board_api_keys", incomingWakeWorkQuery("board_api_keys")!),
// Coalescing consumes a request into another run. Retain it as work until
// both the request and that exact company/agent's run have finished.
workCheck("agent_wakeup_requests", `SELECT 1 FROM agent_wakeup_requests w WHERE
@@ -137,20 +141,7 @@ const WORK_CHECKS = [
// An unconfigured Slack draft has no provider identity or inbound runtime.
// Setup still wakes through normal browser admission. Retain every enabled,
// previously active, partially connected, foreign, or unsupported endpoint.
workCheck("chat_endpoints", `SELECT 1 FROM chat_endpoints endpoint WHERE (endpoint.provider = 'slack'
AND endpoint.status = 'draft' AND endpoint.setup->>'step' = 'provider_setup'
AND (endpoint.setup - ARRAY['step', 'command']) = '{}'::jsonb
AND (endpoint.setup->'command' IS NULL OR endpoint.setup->'command' = 'null'::jsonb
OR jsonb_typeof(endpoint.setup->'command') = 'string')
AND endpoint.activated_at IS NULL AND endpoint.provider_account_id IS NULL
AND endpoint.bot_external_id IS NULL AND endpoint.bot_username IS NULL
AND endpoint.last_event_at IS NULL AND endpoint.last_publication_at IS NULL
AND endpoint.health_message IS NULL AND endpoint.last_error IS NULL
AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id)
AND EXISTS (SELECT 1 FROM tool_connections draft_connection
WHERE draft_connection.id = endpoint.connection_id AND draft_connection.company_id = endpoint.company_id
AND draft_connection.connection_purpose = 'channel' AND draft_connection.transport = 'chat_sdk'
AND draft_connection.status = 'draft' AND draft_connection.enabled = false)) IS NOT TRUE`),
workCheck("chat_endpoints", incomingWakeWorkQuery("chat_endpoints")!),
workCheck("routines", `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`),
workCheck("routine_runs", `SELECT 1 FROM routine_runs WHERE status NOT IN ('completed', 'failed', 'skipped', 'cancelled')`),
workCheck("plugin_jobs", `SELECT 1 FROM plugin_jobs WHERE status <> 'paused'`),
@@ -159,13 +150,7 @@ const WORK_CHECKS = [
// Saved outbound MCP configuration and disabled drafts have no independent
// listener. Keep local processes, channels and GitHub webhook bindings awake;
// accepted calls, credential recovery and delivery work have separate fences.
workCheck("tool_connections", `SELECT 1 FROM tool_connections c WHERE c.status <> 'archived' AND (
(c.status = 'draft' AND c.enabled = false)
OR (c.status = 'active' AND c.connection_purpose = 'ai' AND c.transport = 'runtime_auth')
OR (c.status = 'active' AND c.connection_purpose = 'tool' AND c.transport = 'mcp_remote'
AND NOT EXISTS (SELECT 1 FROM connection_grants g WHERE g.connection_id = c.id
AND (g.company_id <> c.company_id OR g.provider_tenant->'github' IS NOT NULL)))
) IS NOT TRUE`),
workCheck("tool_connections", incomingWakeWorkQuery("tool_connections")!),
// An explicit board escalation has no automatic retry. Retain the visible
// action and all execution/cleanup checks while the board decides what to do.
workCheck("issue_recovery_actions", `SELECT 1 FROM issue_recovery_actions a WHERE a.status NOT IN ('resolved', 'cancelled')
@@ -294,6 +279,8 @@ function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus
* Only a caller holding the exact idle owner may receive an empty report.
*/
export interface IdleSleepSafetyOptions {
incomingWake?: VerifiedIdleCapabilities;
deadlineCapability?: VerifiedIdleCapabilities;
collectDiagnostics?: true;
}
@@ -310,12 +297,15 @@ export async function readIdleSleepSafety(
scheduledIssueMonitors = false,
options: IdleSleepSafetyOptions = {},
): Promise<IdleSleepSafety> {
let incomingProof: IdleIncomingWakeProof | undefined;
const diagnostic = options.collectDiagnostics ? createIdleSleepDiagnostics() : undefined;
const report = (backgroundWork: IdleLocalWork, scheduledWake?: ScheduledWake): IdleSleepSafety => {
const diagnostics = diagnostic?.report();
if (backgroundWork === "none" && diagnostics?.checks.some(check =>
check.state === "unknown" || check.state === "not_checked")) backgroundWork = "unknown";
return { version: backgroundWork === "none" && scheduledWake ? 2 : 1, backgroundWork,
return { version: backgroundWork === "none" && incomingProof ? 3 : backgroundWork === "none" && scheduledWake ? 2 : 1, backgroundWork,
...(backgroundWork === "none" && incomingProof ? { incomingWake: incomingProof.capability,
...(incomingProof.apiWakeAdmission ? { apiWakeAdmission: incomingProof.apiWakeAdmission } : {}) } : {}),
...(backgroundWork === "none" && scheduledWake ? { scheduledWake } : {}),
...(diagnostics ? { diagnostics } : {}) };
};
@@ -360,9 +350,14 @@ export async function readIdleSleepSafety(
let scheduledWake: ScheduledWake | undefined;
const blocked = await db.transaction(async (tx) => {
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
if (options.incomingWake?.incomingWake) {
if (!ownerId || ownerId !== before.ownerId) return undefined;
incomingProof = await readIdleIncomingWakeProof(tx, { capabilities: options.incomingWake, ownerId, receiverOwner: db });
}
const checks = WORK_CHECKS.filter(check =>
(!scheduledRoutines || check.reason !== "routines")
&& (!scheduledIssueMonitors || check.reason !== "issues"));
&& (!scheduledIssueMonitors || check.reason !== "issues"))
.map(check => ({ ...check, query: incomingWakeWorkQuery(check.reason, incomingProof) ?? check.query }));
checks.push(workCheck("agents.company_hold", sql`SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id
WHERE ${companyHoldMismatch()}`));
checks.push(workCheck("heartbeat_runs.budget_cancellation", sql`SELECT 1 FROM heartbeat_runs WHERE ${pendingBudgetRunCancellationPredicate()}`));
@@ -447,6 +442,9 @@ export async function readIdleSleepSafety(
diagnostic?.record("inspection.hold", "unknown");
return report("present");
}
// Legacy alarm kinds cannot be silently combined with a new ingress proof.
// The negotiated deadline catalog supplies the composed v3 alarm separately.
if (incomingProof && scheduledWake) return report("unknown");
return report(local, scheduledWake);
} catch {
// Missing migrations, unknown state, malformed configuration, timeouts and