From 4a334c8bc61064450cb041161cfe34719f7e8623 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 11:39:31 -0600 Subject: [PATCH] Bind idle ingress eligibility to verified wake paths Keep opaque key authentication in the tenant, retain successful API activity through the local safety probe, and inspect exact GitHub broker binding ownership without polling or exempting credential continuity work. Co-Authored-By: Paperclip --- .../src/__tests__/idle-incoming-wake.test.ts | 118 ++++++++++++++++++ server/src/middleware/auth.ts | 10 +- server/src/routes/instance-settings.ts | 4 +- server/src/services/connection-event-wake.ts | 3 + .../src/services/github-connection-events.ts | 4 +- server/src/services/idle-api-activity.test.ts | 18 +++ server/src/services/idle-api-activity.ts | 8 ++ server/src/services/idle-incoming-wake.ts | 74 +++++++++++ server/src/services/idle-local-work.ts | 3 +- server/src/services/idle-sleep-safety.ts | 25 +++- 10 files changed, 259 insertions(+), 8 deletions(-) create mode 100644 server/src/__tests__/idle-incoming-wake.test.ts create mode 100644 server/src/services/idle-api-activity.test.ts create mode 100644 server/src/services/idle-api-activity.ts create mode 100644 server/src/services/idle-incoming-wake.ts diff --git a/server/src/__tests__/idle-incoming-wake.test.ts b/server/src/__tests__/idle-incoming-wake.test.ts new file mode 100644 index 00000000000..e5685b51171 --- /dev/null +++ b/server/src/__tests__/idle-incoming-wake.test.ts @@ -0,0 +1,118 @@ +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, agentApiKeys, authUsers, boardApiKeys, companies, companyMemberships, createDb, toolApplications, toolConnections, connectionGrants, type Db } from "@paperclipai/db"; +import { actorMiddleware } from "../middleware/auth.js"; +import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "../middleware/idle-admission.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { companyRoutes } from "../routes/companies.js"; +import { agentService } from "../services/agents.js"; +import { boardAuthService } from "../services/board-auth.js"; +import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { readIdleIncomingWakeProof, incomingWakeFingerprint } from "../services/idle-incoming-wake.js"; +import { registerConnectionEventWake } from "../services/connection-event-wake.js"; +import { idleApiActivity } from "../services/idle-api-activity.js"; +import { startTaskDrain, stopTaskDrain } from "../services/task-admission.js"; +import { verifyCloudControlAssertion, CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from "../services/cloud-runtime-identity.js"; +import type { IdleIncomingWakeCapability, VerifiedIdleCapabilities } from "../services/idle-incoming-wake-capability.js"; + +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +const url = resolveExternalTestDatabaseUrl("PAPERCLIP_INCOMING_WAKE_TEST_DATABASE_URL"); +const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const ownerId = "00000000-0000-4000-8000-000000000001"; +const now = Date.now(); +const hold = () => ({ ownerId, draining:true, startedAt:new Date(now-1000), expiresAt:new Date(now+300_000), activeRuns:0, pendingWakes:0 }); +const capability: IdleIncomingWakeCapability = { version:1,ownerId,generation:"a".repeat(64),routes:["tenant-api-http-v1"] }; +function verified(incomingWake = capability): VerifiedIdleCapabilities { + const pair=generateKeyPairSync("ed25519"), kid=randomUUID(), issued=Math.floor(Date.now()/1000); + const encode=(v:unknown)=>Buffer.from(JSON.stringify(v)).toString("base64url"); + const header=encode({alg:"EdDSA",typ:CLOUD_CONTROL_JWS_TYPE,kid}); + const payload=encode({v:1,iss:CLOUD_RUNTIME_IDENTITY_ISSUER,aud:CLOUD_CONTROL_AUDIENCE,sub:"fixture-stack",action:"task-drain:read",requestId:randomUUID(),iat:issued,exp:issued+60,idleCapabilities:{incomingWake}}); + const token=`${header}.${payload}.${sign(null,Buffer.from(`${header}.${payload}`),pair.privateKey).toString("base64url")}`; + return verifyCloudControlAssertion({compactJws:token,expectedAction:"task-drain:read",env:{PAPERCLIP_CLOUD_STACK_ID:"fixture-stack",PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS:JSON.stringify({keys:[{...pair.publicKey.export({format:"jwk"}),kid,use:"sig",alg:"EdDSA"}]})}}).idleCapabilities as VerifiedIdleCapabilities; +} + +(support.supported ? describe : describe.skip)("scoped incoming wake PostgreSQL contract",()=>{ + let db: Db; + let database: Awaited> | undefined; + beforeAll(async()=>{ + if (!url) database = await startEmbeddedPostgresTestDatabase("idle-incoming-wake-"); + db=createDb(url ?? database!.connectionString,{maxConnections:4}); + await db.execute(sql`SELECT 1 FROM agent_api_keys LIMIT 1`); // Coordinator migrates the fresh fixture; no embedded fallback. + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN","fixture-only"); + }); + afterAll(async()=>{stopTaskDrain();vi.restoreAllMocks();vi.unstubAllEnvs();await db?.$client.end({timeout:5});await database?.cleanup();}); + const read=(incomingWake?:VerifiedIdleCapabilities)=>readIdleSleepSafety(db,hold,()=>now,ownerId,async()=>"none",undefined,undefined,false,false,{incomingWake}); + async function seed(){ + const companyId=randomUUID(),agentId=randomUUID(),userId=randomUUID(),foreignId=randomUUID(); + await db.insert(companies).values([{id:companyId,name:"Incoming fixture",issuePrefix:`I${companyId.slice(0,6)}`},{id:foreignId,name:"Foreign fixture",issuePrefix:`F${foreignId.slice(0,6)}`}]); + await db.insert(authUsers).values({id:userId,name:"Fixture owner",email:`${userId}@example.test`,createdAt:new Date(),updatedAt:new Date()}); + await db.insert(companyMemberships).values({companyId,principalType:"user",principalId:userId,membershipRole:"owner",status:"active"}); + await db.insert(agents).values({id:agentId,companyId,name:"Fixture agent",role:"engineer",adapterType:"process",status:"idle"}); + const board=await boardAuthService(db).createNamedBoardApiKey({userId,name:"incoming board",expiresAt:null}); + const agent=await agentService(db).createApiKey(agentId,"incoming agent",{kind:"standard"},{responsibleUserId:userId}); + return {companyId,foreignId,userId,agentId,board,agent}; + } + it("retains real saved keys and exempts them only for the verified exact owner/profile",async()=>{ + const f=await seed(); + expect((await read()).backgroundWork).toBe("present"); + expect(await read(verified())).toMatchObject({version:3,backgroundWork:"none",incomingWake:capability}); + expect((await read(verified({...capability,ownerId:randomUUID()}))).backgroundWork).toBe("unknown"); + expect((await db.select().from(agentApiKeys).where(eq(agentApiKeys.id,f.agent.id)))).toHaveLength(1); + expect((await db.select().from(boardApiKeys).where(eq(boardApiKeys.id,f.board.id)))).toHaveLength(1); + }); + it("delivers through actual tenant auth/routes, denies current revoked/expired/foreign authority and holds only successful API use",async()=>{ + const f=await seed(); let monotonic=1; vi.spyOn(performance,"now").mockImplementation(()=>monotonic); + let release:(()=>void)|undefined, arrived:(()=>void)|undefined; + const app=express(); + // Named Cloud wake boundary: preserve request until the simulated compute handoff finishes. + app.use(async(_req,_res,next)=>{if(release){await new Promise(resolve=>{release=resolve;arrived?.();});}next();}); + app.use(idleAdmissionMiddleware);app.use(express.json());app.use(actorMiddleware(db,{deploymentMode:"authenticated",resolveSession:async()=>null})); + app.use("/api/companies",companyRoutes(db));app.use(errorHandler);trackIdleRequestHandlers(app); + expect(idleApiActivity.active()).toBe(false); + expect((await request(app).get(`/api/companies/${f.companyId}`).set("Authorization","Bearer unknown")).status).toBe(401); + expect(idleApiActivity.active()).toBe(false); + expect((await request(app).get(`/api/companies/${f.foreignId}`).set("Authorization",`Bearer ${f.board.token}`)).status).toBe(403); + expect(idleApiActivity.active()).toBe(false); + const own=await request(app).patch(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${f.board.token}`).send({name:"Persisted after wake"}); + expect(own.status).toBe(200);expect(idleApiActivity.active()).toBe(true); + expect((await db.select().from(companies).where(eq(companies.id,f.companyId)))[0]!.name).toBe("Persisted after wake"); + monotonic+=300_000; + expect((await request(app).get(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${f.agent.token}`)).status).toBe(200); + monotonic+=300_001;expect(idleApiActivity.active()).toBe(true);monotonic+=299_999;expect(idleApiActivity.active()).toBe(false); + const arrival=new Promise(resolve=>{arrived=resolve;});release=()=>{}; + const held=request(app).patch(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${f.board.token}`).send({name:"Must not commit"}).then(r=>r); + await arrival;await db.update(boardApiKeys).set({revokedAt:new Date()}).where(eq(boardApiKeys.id,f.board.id));release!();release=undefined; + expect((await held).status).toBe(401);expect(idleApiActivity.active()).toBe(false); + const expired=await boardAuthService(db).createNamedBoardApiKey({userId:f.userId,name:"expired",expiresAt:new Date(0)}); + expect((await request(app).get(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${expired.token}`)).status).toBe(401);expect(idleApiActivity.active()).toBe(false); + startTaskDrain({purpose:"idle",ttlMs:60_000}); + expect((await request(app).patch(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${f.agent.token}`).send({name:"Draining"})).status).toBe(503); + expect(idleApiActivity.active()).toBe(false);stopTaskDrain(); + expect((await db.select().from(companies).where(eq(companies.id,f.companyId)))[0]!.name).toBe("Persisted after wake"); + }); + it("requires current broker instance and exact owner binding, preserving continuity and unsupported transport blockers",async()=>{ + const f=await seed();const applicationId=randomUUID(),connectionId=randomUUID(),grantId=randomUUID(); + await db.insert(toolApplications).values({id:applicationId,companyId:f.companyId,name:"GitHub",type:"mcp_http"}); + await db.insert(toolConnections).values({id:connectionId,companyId:f.companyId,applicationId,name:"GitHub",uid:randomUUID(),transport:"mcp_remote",connectionPurpose:"tool",status:"active",enabled:true,config:{sourceTemplateKey:"github",oauth:{connectorProfile:"github.code"}}}); + const github={userId:"42",login:"fixture",installationCount:1,repositoryCount:1,repositorySelection:"all" as const,installationIds:["123"],installationOwnerLogins:["fixture"],lastAccessRefreshAt:new Date().toISOString()}; + await db.insert(connectionGrants).values({id:grantId,companyId:f.companyId,connectionId,kind:"user",subjectUserId:f.userId,providerTenant:{github},status:"active"}); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID","fixture-instance");vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY","fixture");vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY","fixture");vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT","test");vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL","http://127.0.0.1"); + const stop=registerConnectionEventWake(db,()=>{}); + const brokerCap:IdleIncomingWakeCapability={...capability,routes:["github-broker-events-v1","tenant-api-http-v1"],githubBroker:{instanceId:"fixture-instance",bindingsFingerprint:incomingWakeFingerprint([[`${grantId}_123`,f.companyId,f.userId,"github.code","123",connectionId,grantId]])}}; + try { + const proof=await readIdleIncomingWakeProof(db,{capabilities:verified(brokerCap),ownerId,receiverOwner:db}); + expect(proof?.requiredContinuityConnectionIds).toEqual([connectionId]);expect(proof?.connectionIds).toEqual([]); + expect((await read(verified(brokerCap))).backgroundWork).toBe("present"); // transport alone never discharges real credential deadline + await db.update(connectionGrants).set({providerTenant:{github:{...github,bindingSync:{id:randomUUID(),removeInstallationIds:[]}}}}).where(eq(connectionGrants.id,grantId)); + expect((await read(verified(brokerCap))).backgroundWork).toBe("unknown"); + await db.update(connectionGrants).set({status:"revoked"}).where(eq(connectionGrants.id,grantId)); + expect((await read(verified(brokerCap))).backgroundWork).toBe("unknown"); + await db.update(toolConnections).set({transport:"local_stdio"}).where(eq(toolConnections.id,connectionId)); + expect((await read(verified())).backgroundWork).toBe("present"); + } finally {stop();} + }); +}); diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index f9bd7446947..bf8a05f22d0 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -1,3 +1,4 @@ +import { idleApiActivity } from "../services/idle-api-activity.js"; import { createHash, timingSafeEqual } from "node:crypto"; import type { Request, RequestHandler } from "express"; import { and, eq, isNull, sql } from "drizzle-orm"; @@ -43,7 +44,7 @@ import { ensureHumanRoleDefaultGrants } from "../services/principal-access-compa import { forbidden, unauthorized, unprocessable } from "../errors.js"; export { isCloudManagedInstance } from "../services/cloud-instance.js"; -import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js"; +import { cloudTenantPrimaryCompanyId, isCloudManagedInstance } from "../services/cloud-instance.js"; function hashToken(token: string) { return createHash("sha256").update(token).digest("hex"); @@ -329,6 +330,9 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa runId: runIdHeader || undefined, source: "board_key", }; + if (isCloudManagedInstance() && (access.isInstanceAdmin || access.companyIds.length)) { + _res.once("finish", () => { if (_res.statusCode >= 200 && _res.statusCode < 400) idleApiActivity.record(); }); + } next(); return; } @@ -521,7 +525,9 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa runId: runIdHeader || undefined, source: "agent_key", }; - + if (isCloudManagedInstance() && req.actor.onBehalfOfMemberships?.length) { + _res.once("finish", () => { if (_res.statusCode >= 200 && _res.statusCode < 400) idleApiActivity.record(); }); + } next(); }; return async (req, res, next) => { diff --git a/server/src/routes/instance-settings.ts b/server/src/routes/instance-settings.ts index b8072b8b308..c768c3f88e2 100644 --- a/server/src/routes/instance-settings.ts +++ b/server/src/routes/instance-settings.ts @@ -1,3 +1,4 @@ +import { verifiedIdleCapabilities } from "../middleware/cloud-control.js"; import { trackIdleWork } from "../services/task-admission.js"; import { Router, type Request } from "express"; import { companies, type Db } from "@paperclipai/db"; @@ -335,7 +336,8 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag const idleSleepSafety = await readIdleSleepSafety(db, () => heartbeat.getTaskDrainStatus(), Date.now, typeof req.query.ownerId === "string" ? req.query.ownerId : undefined, () => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup) }), - pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors); + pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors, + { incomingWake: verifiedIdleCapabilities(req), deadlineCapability: verifiedIdleCapabilities(req) }); res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety }); return; } diff --git a/server/src/services/connection-event-wake.ts b/server/src/services/connection-event-wake.ts index 87840eb0979..cd7bd24da51 100644 --- a/server/src/services/connection-event-wake.ts +++ b/server/src/services/connection-event-wake.ts @@ -10,3 +10,6 @@ export function wakeConnectionEvents(owner: object): boolean { wake(); return true; } + +/** Inspection never emits a wake or polls the broker. */ +export const hasConnectionEventWakeReceiver = (owner: object): boolean => receivers.has(owner); diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index af936353bdd..e84092ed826 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -146,7 +146,7 @@ function normalizeLeasedPayload(event: LeasedEvent): Record { return compact(base); } -function bindingRows(rows: Array<{ +export function githubConnectionEventBindings(rows: Array<{ grant: typeof connectionGrants.$inferSelect; connection: typeof toolConnections.$inferSelect; }>): GitHubBinding[] { @@ -259,7 +259,7 @@ export function githubConnectionEventService( eq(toolConnections.enabled, true), sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status <> 'archived')`, )); - return bindingRows(rows); + return githubConnectionEventBindings(rows); } async function applyPullRequestEvent(database: Db, companyId: string, event: LeasedEvent, effects: Array<(committedDb: Db) => Promise>) { diff --git a/server/src/services/idle-api-activity.test.ts b/server/src/services/idle-api-activity.test.ts new file mode 100644 index 00000000000..5bf26ad0b17 --- /dev/null +++ b/server/src/services/idle-api-activity.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { IdleApiActivity } from "./idle-api-activity.js"; + +describe("authenticated API idle activity", () => { + it("is passive without successful authentication and expires without timers or dependency work", () => { + let monotonic = 1; const activity = new IdleApiActivity(() => monotonic); + expect(activity.active()).toBe(false); + activity.record(); expect(activity.active()).toBe(true); + monotonic += 599_999; expect(activity.active()).toBe(true); + monotonic++; expect(activity.active()).toBe(false); + }); + it("extends only from the actual latest successful use and ignores wall clock changes", () => { + let monotonic = 1; const activity = new IdleApiActivity(() => monotonic); + activity.record(); monotonic += 300_000; activity.record(); + monotonic += 300_001; expect(activity.active()).toBe(true); + monotonic += 299_999; expect(activity.active()).toBe(false); + }); +}); diff --git a/server/src/services/idle-api-activity.ts b/server/src/services/idle-api-activity.ts new file mode 100644 index 00000000000..553753f99cd --- /dev/null +++ b/server/src/services/idle-api-activity.ts @@ -0,0 +1,8 @@ +/** Authenticated API use is real activity; rejecting an unknown key is not. */ +export class IdleApiActivity { + private until = 0; + constructor(private readonly now: () => number = () => performance.now()) {} + record(): void { this.until = Math.max(this.until, this.now() + 10 * 60_000); } + active(): boolean { return this.now() < this.until; } +} +export const idleApiActivity = new IdleApiActivity(); diff --git a/server/src/services/idle-incoming-wake.ts b/server/src/services/idle-incoming-wake.ts new file mode 100644 index 00000000000..b4c714486aa --- /dev/null +++ b/server/src/services/idle-incoming-wake.ts @@ -0,0 +1,74 @@ +import { createHash } from "node:crypto"; +import { and, eq, sql, type SQL } from "drizzle-orm"; +import { chatEndpoints, chatGitHubRegistrations, connectionGrants, toolConnections, type Db } from "@paperclipai/db"; +import type { IdleIncomingWakeCapability, VerifiedIdleCapabilities } from "./idle-incoming-wake-capability.js"; +import { githubConnectionEventBindings } from "./github-connection-events.js"; +import { paperclipCloudConnectorConfigFromEnv } from "./paperclip-cloud-connector.js"; +import { hasConnectionEventWakeReceiver } from "./connection-event-wake.js"; + +export interface IdleIncomingWakeProof { + capability: IdleIncomingWakeCapability; + connectionIds: string[]; + endpointIds: string[]; + /** A broker enrollment cannot discharge these credential continuity obligations. */ + requiredContinuityConnectionIds: string[]; +} +export const incomingWakeFingerprint = (bindings: string[][]) => createHash("sha256").update(JSON.stringify(bindings.map(b => [...b]).sort((a,b) => JSON.stringify(a) < JSON.stringify(b) ? -1 : JSON.stringify(a) > JSON.stringify(b) ? 1 : 0))).digest("hex"); + +/** Read current binding authority in the safety transaction. No network call, renewal or write. */ +export async function readIdleIncomingWakeProof(tx: Pick, input: { + capabilities?: VerifiedIdleCapabilities; ownerId: string; receiverOwner: object; +}): Promise { + const capability = input.capabilities?.incomingWake; + if (!capability) return; + if (capability.ownerId !== input.ownerId) throw new Error("Incoming wake owner changed"); + const proof: IdleIncomingWakeProof = { capability, connectionIds: [], endpointIds: [], requiredContinuityConnectionIds: [] }; + if (!capability.routes.includes("github-broker-events-v1")) return proof; + const broker = capability.githubBroker; + const config = paperclipCloudConnectorConfigFromEnv(); + if (!broker || config?.instanceId !== broker.instanceId || !hasConnectionEventWakeReceiver(input.receiverOwner)) + throw new Error("Incoming GitHub receiver is not bound"); + const rows = await tx.select({ grant: connectionGrants, connection: toolConnections }).from(connectionGrants) + .innerJoin(toolConnections, and(eq(toolConnections.id, connectionGrants.connectionId), eq(toolConnections.companyId, connectionGrants.companyId))) + .where(and(eq(connectionGrants.status, "active"), eq(toolConnections.status, "active"), eq(toolConnections.enabled, true), + sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`, + sql`exists (select 1 from companies where id=${connectionGrants.companyId} and status <> 'archived')`)); + if (rows.length > 1000) throw new Error("Too many GitHub bindings to inspect"); + for (const { grant, connection } of rows) { + const github = grant.providerTenant?.github; + if (!github || !Array.isArray(github.installationIds) || !github.installationIds.length || github.bindingSync || github.continuityRetry + || github.installationIds.some(id => typeof id !== "string" || !/^[0-9]+$/.test(id))) throw new Error("Unsettled GitHub authority"); + proof.requiredContinuityConnectionIds.push(connection.id); + } + const bindings = githubConnectionEventBindings(rows); + if (bindings.length !== rows.reduce((n,r) => n + r.grant.providerTenant!.github!.installationIds.length, 0)) throw new Error("Incomplete GitHub binding"); + const tuples = bindings.map(b => [b.id,b.companyId,b.subject,"github.code",b.installationId,b.connectionId,b.grantId]); + const endpoints = await tx.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints) + .innerJoin(toolConnections, and(eq(toolConnections.id,chatEndpoints.connectionId),eq(toolConnections.companyId,chatEndpoints.companyId))) + .where(and(eq(chatEndpoints.provider,"github"),eq(chatEndpoints.status,"active"),eq(toolConnections.status,"active"),eq(toolConnections.enabled,true), + eq(toolConnections.transport,"chat_sdk"),eq(toolConnections.connectionPurpose,"channel"), + sql`exists (select 1 from companies where id=${chatEndpoints.companyId} and status <> 'archived')`)); + for (const { endpoint, connection } of endpoints) { + const registration = endpoint.setup.github?.cloudRegistrationId; + if (!registration || !/^[A-Za-z0-9_-]{43}$/.test(registration)) continue; + const registrations = await tx.select({ userId: chatGitHubRegistrations.userId }).from(chatGitHubRegistrations) + .where(and(eq(chatGitHubRegistrations.endpointId, endpoint.id), eq(chatGitHubRegistrations.companyId, endpoint.companyId), + eq(chatGitHubRegistrations.status, "completed"), sql`${chatGitHubRegistrations.handoff}->>'cloudId' = ${registration}`)).limit(2); + if (registrations.length !== 1 || !registrations[0]!.userId) throw new Error("GitHub registration authority changed"); + tuples.push([`github-app:${registration}`,endpoint.companyId,registrations[0]!.userId,"github.bot",`gateway:${registration}`,registration,registration]); + proof.connectionIds.push(connection.id); proof.endpointIds.push(endpoint.id); + } + if (incomingWakeFingerprint(tuples) !== broker.bindingsFingerprint) throw new Error("GitHub wake bindings changed"); + return proof; +} + +/** Narrowly substitute proven rows while leaving every other ingress predicate intact. */ +export function incomingWakeWorkCheck(query: string, proof?: IdleIncomingWakeProof, continuityConnectionIds: readonly string[] = []): SQL { + if (proof?.capability.routes.includes("tenant-api-http-v1") && (query.startsWith("SELECT 1 FROM agent_api_keys ") || query.startsWith("SELECT 1 FROM board_api_keys "))) return sql`false`; + const endpointPrefix = "SELECT 1 FROM chat_endpoints endpoint WHERE "; + const connectionPrefix = "SELECT 1 FROM tool_connections c WHERE "; + const connectionIds = [...proof?.connectionIds ?? [], ...proof?.requiredContinuityConnectionIds.filter(id => continuityConnectionIds.includes(id)) ?? []]; + if (proof?.endpointIds.length && query.startsWith(endpointPrefix)) return sql`EXISTS (SELECT 1 FROM chat_endpoints endpoint WHERE endpoint.id NOT IN (${sql.join(proof.endpointIds.map(id => sql`${id}::uuid`),sql`, `)}) AND (${sql.raw(query.slice(endpointPrefix.length))}))`; + if (connectionIds.length && query.startsWith(connectionPrefix)) return sql`EXISTS (SELECT 1 FROM tool_connections c WHERE c.id NOT IN (${sql.join(connectionIds.map(id => sql`${id}::uuid`),sql`, `)}) AND (${sql.raw(query.slice(connectionPrefix.length))}))`; + return sql`EXISTS (${sql.raw(query)})`; +} diff --git a/server/src/services/idle-local-work.ts b/server/src/services/idle-local-work.ts index 95b6549ebbc..0cd25aae858 100644 --- a/server/src/services/idle-local-work.ts +++ b/server/src/services/idle-local-work.ts @@ -1,3 +1,4 @@ +import { idleApiActivity } from "./idle-api-activity.js"; import { promises as fs } from "node:fs"; import path from "node:path"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; @@ -65,7 +66,7 @@ export async function readIdleLocalWork(options: { backupCheckpoint?: boolean } if (!startupComplete || !ingressTracked) return "unknown"; // This exemption belongs only to a scan which will finish a new backup // under the same admission hold before it can return an empty report. - if ((scheduledBackups && !options.backupCheckpoint) || idleWorkSnapshot().active !== 0) return "present"; + if (idleApiActivity.active() || (scheduledBackups && !options.backupCheckpoint) || idleWorkSnapshot().active !== 0) return "present"; try { const local = [...localWorkProbes].map(probe => probe()); if (local.includes("unknown")) return "unknown"; diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index c0d58a64bdd..59f08256b64 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -1,3 +1,5 @@ +import { readIdleIncomingWakeProof, incomingWakeWorkCheck, type IdleIncomingWakeProof } from "./idle-incoming-wake.js"; +import type { VerifiedIdleCapabilities, IdleIncomingWakeCapability } from "./idle-incoming-wake-capability.js"; import { idleWorkSnapshot } from "./task-admission.js"; import { readIdleLocalWork, type IdleLocalWork } from "./idle-local-work.js"; import { readIdleRoutineSchedule, type ScheduledWake } from "./idle-scheduled-work.js"; @@ -20,9 +22,16 @@ export interface IdleSleepDrainStatus { } export interface IdleSleepSafety { - version: 1 | 2; + version: 1 | 2 | 3; backgroundWork: "none" | "present" | "unknown"; scheduledWake?: ScheduledWake; + incomingWake?: IdleIncomingWakeCapability; +} + +export interface IdleSleepSafetyOptions { + incomingWake?: VerifiedIdleCapabilities; + deadlineCapability?: VerifiedIdleCapabilities; + collectDiagnostics?: true; } export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) => Promise<{ @@ -298,6 +307,7 @@ export async function readIdleSleepSafety( prepareBackup?: () => Promise, scheduledRoutines = false, scheduledIssueMonitors = false, + options: IdleSleepSafetyOptions = {}, ): Promise { const unknown: IdleSleepSafety = { version: 1, backgroundWork: "unknown" }; const before = getDrainStatus(); @@ -316,12 +326,17 @@ export async function readIdleSleepSafety( quietPlugins = pluginWork.pluginIds; } let scheduledWake: ScheduledWake | undefined; + let incomingProof: IdleIncomingWakeProof | undefined; const blocked = await db.transaction(async (tx) => { await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + if (options.incomingWake?.incomingWake) { + if (!ownerId || ownerId !== before.ownerId) return undefined; + incomingProof = await readIdleIncomingWakeProof(tx, { capabilities: options.incomingWake, ownerId, receiverOwner: db }); + } const checks = WORK_CHECKS.filter(query => (!scheduledRoutines || query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`) && (!scheduledIssueMonitors || query !== `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`)) - .map((query) => sql`EXISTS (${sql.raw(query)})`); + .map((query) => incomingWakeWorkCheck(query, incomingProof)); checks.push(sql`EXISTS (SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id WHERE ${companyHoldMismatch()})`); checks.push(sql`EXISTS (SELECT 1 FROM heartbeat_runs WHERE ${pendingBudgetRunCancellationPredicate()})`); @@ -366,6 +381,12 @@ export async function readIdleSleepSafety( const localAfter = idleWorkSnapshot(); if (!sameQuietHold(before, after, now()) || localAfter.active !== 0 || expectedGeneration !== localAfter.generation) return unknown; + if (local === "none" && incomingProof) { + // Until the negotiated deadline catalog owns the combined proof, retain + // legacy scheduled obligations rather than silently omit their alarm. + if (scheduledWake) return unknown; + return { version: 3, backgroundWork: "none", incomingWake: incomingProof.capability }; + } if (local === "none" && scheduledWake) { if (Date.parse(scheduledWake.nextWakeAt) <= now()) return { version: 1, backgroundWork: "present" }; return { version: 2, backgroundWork: "none", scheduledWake };