diff --git a/server/src/__tests__/idle-incoming-wake.test.ts b/server/src/__tests__/idle-incoming-wake.test.ts new file mode 100644 index 00000000000..e5685b51171 --- /dev/null +++ b/server/src/__tests__/idle-incoming-wake.test.ts @@ -0,0 +1,118 @@ +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, agentApiKeys, authUsers, boardApiKeys, companies, companyMemberships, createDb, toolApplications, toolConnections, connectionGrants, type Db } from "@paperclipai/db"; +import { actorMiddleware } from "../middleware/auth.js"; +import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "../middleware/idle-admission.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { companyRoutes } from "../routes/companies.js"; +import { agentService } from "../services/agents.js"; +import { boardAuthService } from "../services/board-auth.js"; +import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { readIdleIncomingWakeProof, incomingWakeFingerprint } from "../services/idle-incoming-wake.js"; +import { registerConnectionEventWake } from "../services/connection-event-wake.js"; +import { idleApiActivity } from "../services/idle-api-activity.js"; +import { startTaskDrain, stopTaskDrain } from "../services/task-admission.js"; +import { verifyCloudControlAssertion, CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_JWS_TYPE, CLOUD_RUNTIME_IDENTITY_ISSUER } from "../services/cloud-runtime-identity.js"; +import type { IdleIncomingWakeCapability, VerifiedIdleCapabilities } from "../services/idle-incoming-wake-capability.js"; + +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +const url = resolveExternalTestDatabaseUrl("PAPERCLIP_INCOMING_WAKE_TEST_DATABASE_URL"); +const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const ownerId = "00000000-0000-4000-8000-000000000001"; +const now = Date.now(); +const hold = () => ({ ownerId, draining:true, startedAt:new Date(now-1000), expiresAt:new Date(now+300_000), activeRuns:0, pendingWakes:0 }); +const capability: IdleIncomingWakeCapability = { version:1,ownerId,generation:"a".repeat(64),routes:["tenant-api-http-v1"] }; +function verified(incomingWake = capability): VerifiedIdleCapabilities { + const pair=generateKeyPairSync("ed25519"), kid=randomUUID(), issued=Math.floor(Date.now()/1000); + const encode=(v:unknown)=>Buffer.from(JSON.stringify(v)).toString("base64url"); + const header=encode({alg:"EdDSA",typ:CLOUD_CONTROL_JWS_TYPE,kid}); + const payload=encode({v:1,iss:CLOUD_RUNTIME_IDENTITY_ISSUER,aud:CLOUD_CONTROL_AUDIENCE,sub:"fixture-stack",action:"task-drain:read",requestId:randomUUID(),iat:issued,exp:issued+60,idleCapabilities:{incomingWake}}); + const token=`${header}.${payload}.${sign(null,Buffer.from(`${header}.${payload}`),pair.privateKey).toString("base64url")}`; + return verifyCloudControlAssertion({compactJws:token,expectedAction:"task-drain:read",env:{PAPERCLIP_CLOUD_STACK_ID:"fixture-stack",PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS:JSON.stringify({keys:[{...pair.publicKey.export({format:"jwk"}),kid,use:"sig",alg:"EdDSA"}]})}}).idleCapabilities as VerifiedIdleCapabilities; +} + +(support.supported ? describe : describe.skip)("scoped incoming wake PostgreSQL contract",()=>{ + let db: Db; + let database: Awaited> | undefined; + beforeAll(async()=>{ + if (!url) database = await startEmbeddedPostgresTestDatabase("idle-incoming-wake-"); + db=createDb(url ?? database!.connectionString,{maxConnections:4}); + await db.execute(sql`SELECT 1 FROM agent_api_keys LIMIT 1`); // Coordinator migrates the fresh fixture; no embedded fallback. + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN","fixture-only"); + }); + afterAll(async()=>{stopTaskDrain();vi.restoreAllMocks();vi.unstubAllEnvs();await db?.$client.end({timeout:5});await database?.cleanup();}); + const read=(incomingWake?:VerifiedIdleCapabilities)=>readIdleSleepSafety(db,hold,()=>now,ownerId,async()=>"none",undefined,undefined,false,false,{incomingWake}); + async function seed(){ + const companyId=randomUUID(),agentId=randomUUID(),userId=randomUUID(),foreignId=randomUUID(); + await db.insert(companies).values([{id:companyId,name:"Incoming fixture",issuePrefix:`I${companyId.slice(0,6)}`},{id:foreignId,name:"Foreign fixture",issuePrefix:`F${foreignId.slice(0,6)}`}]); + await db.insert(authUsers).values({id:userId,name:"Fixture owner",email:`${userId}@example.test`,createdAt:new Date(),updatedAt:new Date()}); + await db.insert(companyMemberships).values({companyId,principalType:"user",principalId:userId,membershipRole:"owner",status:"active"}); + await db.insert(agents).values({id:agentId,companyId,name:"Fixture agent",role:"engineer",adapterType:"process",status:"idle"}); + const board=await boardAuthService(db).createNamedBoardApiKey({userId,name:"incoming board",expiresAt:null}); + const agent=await agentService(db).createApiKey(agentId,"incoming agent",{kind:"standard"},{responsibleUserId:userId}); + return {companyId,foreignId,userId,agentId,board,agent}; + } + it("retains real saved keys and exempts them only for the verified exact owner/profile",async()=>{ + const f=await seed(); + expect((await read()).backgroundWork).toBe("present"); + expect(await read(verified())).toMatchObject({version:3,backgroundWork:"none",incomingWake:capability}); + expect((await read(verified({...capability,ownerId:randomUUID()}))).backgroundWork).toBe("unknown"); + expect((await db.select().from(agentApiKeys).where(eq(agentApiKeys.id,f.agent.id)))).toHaveLength(1); + expect((await db.select().from(boardApiKeys).where(eq(boardApiKeys.id,f.board.id)))).toHaveLength(1); + }); + it("delivers through actual tenant auth/routes, denies current revoked/expired/foreign authority and holds only successful API use",async()=>{ + const f=await seed(); let monotonic=1; vi.spyOn(performance,"now").mockImplementation(()=>monotonic); + let release:(()=>void)|undefined, arrived:(()=>void)|undefined; + const app=express(); + // Named Cloud wake boundary: preserve request until the simulated compute handoff finishes. + app.use(async(_req,_res,next)=>{if(release){await new Promise(resolve=>{release=resolve;arrived?.();});}next();}); + app.use(idleAdmissionMiddleware);app.use(express.json());app.use(actorMiddleware(db,{deploymentMode:"authenticated",resolveSession:async()=>null})); + app.use("/api/companies",companyRoutes(db));app.use(errorHandler);trackIdleRequestHandlers(app); + expect(idleApiActivity.active()).toBe(false); + expect((await request(app).get(`/api/companies/${f.companyId}`).set("Authorization","Bearer unknown")).status).toBe(401); + expect(idleApiActivity.active()).toBe(false); + expect((await request(app).get(`/api/companies/${f.foreignId}`).set("Authorization",`Bearer ${f.board.token}`)).status).toBe(403); + expect(idleApiActivity.active()).toBe(false); + const own=await request(app).patch(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${f.board.token}`).send({name:"Persisted after wake"}); + expect(own.status).toBe(200);expect(idleApiActivity.active()).toBe(true); + expect((await db.select().from(companies).where(eq(companies.id,f.companyId)))[0]!.name).toBe("Persisted after wake"); + monotonic+=300_000; + expect((await request(app).get(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${f.agent.token}`)).status).toBe(200); + monotonic+=300_001;expect(idleApiActivity.active()).toBe(true);monotonic+=299_999;expect(idleApiActivity.active()).toBe(false); + const arrival=new Promise(resolve=>{arrived=resolve;});release=()=>{}; + const held=request(app).patch(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${f.board.token}`).send({name:"Must not commit"}).then(r=>r); + await arrival;await db.update(boardApiKeys).set({revokedAt:new Date()}).where(eq(boardApiKeys.id,f.board.id));release!();release=undefined; + expect((await held).status).toBe(401);expect(idleApiActivity.active()).toBe(false); + const expired=await boardAuthService(db).createNamedBoardApiKey({userId:f.userId,name:"expired",expiresAt:new Date(0)}); + expect((await request(app).get(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${expired.token}`)).status).toBe(401);expect(idleApiActivity.active()).toBe(false); + startTaskDrain({purpose:"idle",ttlMs:60_000}); + expect((await request(app).patch(`/api/companies/${f.companyId}`).set("Authorization",`Bearer ${f.agent.token}`).send({name:"Draining"})).status).toBe(503); + expect(idleApiActivity.active()).toBe(false);stopTaskDrain(); + expect((await db.select().from(companies).where(eq(companies.id,f.companyId)))[0]!.name).toBe("Persisted after wake"); + }); + it("requires current broker instance and exact owner binding, preserving continuity and unsupported transport blockers",async()=>{ + const f=await seed();const applicationId=randomUUID(),connectionId=randomUUID(),grantId=randomUUID(); + await db.insert(toolApplications).values({id:applicationId,companyId:f.companyId,name:"GitHub",type:"mcp_http"}); + await db.insert(toolConnections).values({id:connectionId,companyId:f.companyId,applicationId,name:"GitHub",uid:randomUUID(),transport:"mcp_remote",connectionPurpose:"tool",status:"active",enabled:true,config:{sourceTemplateKey:"github",oauth:{connectorProfile:"github.code"}}}); + const github={userId:"42",login:"fixture",installationCount:1,repositoryCount:1,repositorySelection:"all" as const,installationIds:["123"],installationOwnerLogins:["fixture"],lastAccessRefreshAt:new Date().toISOString()}; + await db.insert(connectionGrants).values({id:grantId,companyId:f.companyId,connectionId,kind:"user",subjectUserId:f.userId,providerTenant:{github},status:"active"}); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID","fixture-instance");vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY","fixture");vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY","fixture");vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT","test");vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL","http://127.0.0.1"); + const stop=registerConnectionEventWake(db,()=>{}); + const brokerCap:IdleIncomingWakeCapability={...capability,routes:["github-broker-events-v1","tenant-api-http-v1"],githubBroker:{instanceId:"fixture-instance",bindingsFingerprint:incomingWakeFingerprint([[`${grantId}_123`,f.companyId,f.userId,"github.code","123",connectionId,grantId]])}}; + try { + const proof=await readIdleIncomingWakeProof(db,{capabilities:verified(brokerCap),ownerId,receiverOwner:db}); + expect(proof?.requiredContinuityConnectionIds).toEqual([connectionId]);expect(proof?.connectionIds).toEqual([]); + expect((await read(verified(brokerCap))).backgroundWork).toBe("present"); // transport alone never discharges real credential deadline + await db.update(connectionGrants).set({providerTenant:{github:{...github,bindingSync:{id:randomUUID(),removeInstallationIds:[]}}}}).where(eq(connectionGrants.id,grantId)); + expect((await read(verified(brokerCap))).backgroundWork).toBe("unknown"); + await db.update(connectionGrants).set({status:"revoked"}).where(eq(connectionGrants.id,grantId)); + expect((await read(verified(brokerCap))).backgroundWork).toBe("unknown"); + await db.update(toolConnections).set({transport:"local_stdio"}).where(eq(toolConnections.id,connectionId)); + expect((await read(verified())).backgroundWork).toBe("present"); + } finally {stop();} + }); +}); diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index f9bd7446947..bf8a05f22d0 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -1,3 +1,4 @@ +import { idleApiActivity } from "../services/idle-api-activity.js"; import { createHash, timingSafeEqual } from "node:crypto"; import type { Request, RequestHandler } from "express"; import { and, eq, isNull, sql } from "drizzle-orm"; @@ -43,7 +44,7 @@ import { ensureHumanRoleDefaultGrants } from "../services/principal-access-compa import { forbidden, unauthorized, unprocessable } from "../errors.js"; export { isCloudManagedInstance } from "../services/cloud-instance.js"; -import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js"; +import { cloudTenantPrimaryCompanyId, isCloudManagedInstance } from "../services/cloud-instance.js"; function hashToken(token: string) { return createHash("sha256").update(token).digest("hex"); @@ -329,6 +330,9 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa runId: runIdHeader || undefined, source: "board_key", }; + if (isCloudManagedInstance() && (access.isInstanceAdmin || access.companyIds.length)) { + _res.once("finish", () => { if (_res.statusCode >= 200 && _res.statusCode < 400) idleApiActivity.record(); }); + } next(); return; } @@ -521,7 +525,9 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa runId: runIdHeader || undefined, source: "agent_key", }; - + if (isCloudManagedInstance() && req.actor.onBehalfOfMemberships?.length) { + _res.once("finish", () => { if (_res.statusCode >= 200 && _res.statusCode < 400) idleApiActivity.record(); }); + } next(); }; return async (req, res, next) => { diff --git a/server/src/routes/instance-settings.ts b/server/src/routes/instance-settings.ts index b8072b8b308..c768c3f88e2 100644 --- a/server/src/routes/instance-settings.ts +++ b/server/src/routes/instance-settings.ts @@ -1,3 +1,4 @@ +import { verifiedIdleCapabilities } from "../middleware/cloud-control.js"; import { trackIdleWork } from "../services/task-admission.js"; import { Router, type Request } from "express"; import { companies, type Db } from "@paperclipai/db"; @@ -335,7 +336,8 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag const idleSleepSafety = await readIdleSleepSafety(db, () => heartbeat.getTaskDrainStatus(), Date.now, typeof req.query.ownerId === "string" ? req.query.ownerId : undefined, () => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup) }), - pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors); + pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors, + { incomingWake: verifiedIdleCapabilities(req), deadlineCapability: verifiedIdleCapabilities(req) }); res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety }); return; } diff --git a/server/src/services/connection-event-wake.ts b/server/src/services/connection-event-wake.ts index 87840eb0979..cd7bd24da51 100644 --- a/server/src/services/connection-event-wake.ts +++ b/server/src/services/connection-event-wake.ts @@ -10,3 +10,6 @@ export function wakeConnectionEvents(owner: object): boolean { wake(); return true; } + +/** Inspection never emits a wake or polls the broker. */ +export const hasConnectionEventWakeReceiver = (owner: object): boolean => receivers.has(owner); diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index af936353bdd..e84092ed826 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -146,7 +146,7 @@ function normalizeLeasedPayload(event: LeasedEvent): Record { return compact(base); } -function bindingRows(rows: Array<{ +export function githubConnectionEventBindings(rows: Array<{ grant: typeof connectionGrants.$inferSelect; connection: typeof toolConnections.$inferSelect; }>): GitHubBinding[] { @@ -259,7 +259,7 @@ export function githubConnectionEventService( eq(toolConnections.enabled, true), sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status <> 'archived')`, )); - return bindingRows(rows); + return githubConnectionEventBindings(rows); } async function applyPullRequestEvent(database: Db, companyId: string, event: LeasedEvent, effects: Array<(committedDb: Db) => Promise>) { diff --git a/server/src/services/idle-api-activity.test.ts b/server/src/services/idle-api-activity.test.ts new file mode 100644 index 00000000000..5bf26ad0b17 --- /dev/null +++ b/server/src/services/idle-api-activity.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { IdleApiActivity } from "./idle-api-activity.js"; + +describe("authenticated API idle activity", () => { + it("is passive without successful authentication and expires without timers or dependency work", () => { + let monotonic = 1; const activity = new IdleApiActivity(() => monotonic); + expect(activity.active()).toBe(false); + activity.record(); expect(activity.active()).toBe(true); + monotonic += 599_999; expect(activity.active()).toBe(true); + monotonic++; expect(activity.active()).toBe(false); + }); + it("extends only from the actual latest successful use and ignores wall clock changes", () => { + let monotonic = 1; const activity = new IdleApiActivity(() => monotonic); + activity.record(); monotonic += 300_000; activity.record(); + monotonic += 300_001; expect(activity.active()).toBe(true); + monotonic += 299_999; expect(activity.active()).toBe(false); + }); +}); diff --git a/server/src/services/idle-api-activity.ts b/server/src/services/idle-api-activity.ts new file mode 100644 index 00000000000..553753f99cd --- /dev/null +++ b/server/src/services/idle-api-activity.ts @@ -0,0 +1,8 @@ +/** Authenticated API use is real activity; rejecting an unknown key is not. */ +export class IdleApiActivity { + private until = 0; + constructor(private readonly now: () => number = () => performance.now()) {} + record(): void { this.until = Math.max(this.until, this.now() + 10 * 60_000); } + active(): boolean { return this.now() < this.until; } +} +export const idleApiActivity = new IdleApiActivity(); diff --git a/server/src/services/idle-incoming-wake.ts b/server/src/services/idle-incoming-wake.ts new file mode 100644 index 00000000000..b4c714486aa --- /dev/null +++ b/server/src/services/idle-incoming-wake.ts @@ -0,0 +1,74 @@ +import { createHash } from "node:crypto"; +import { and, eq, sql, type SQL } from "drizzle-orm"; +import { chatEndpoints, chatGitHubRegistrations, connectionGrants, toolConnections, type Db } from "@paperclipai/db"; +import type { IdleIncomingWakeCapability, VerifiedIdleCapabilities } from "./idle-incoming-wake-capability.js"; +import { githubConnectionEventBindings } from "./github-connection-events.js"; +import { paperclipCloudConnectorConfigFromEnv } from "./paperclip-cloud-connector.js"; +import { hasConnectionEventWakeReceiver } from "./connection-event-wake.js"; + +export interface IdleIncomingWakeProof { + capability: IdleIncomingWakeCapability; + connectionIds: string[]; + endpointIds: string[]; + /** A broker enrollment cannot discharge these credential continuity obligations. */ + requiredContinuityConnectionIds: string[]; +} +export const incomingWakeFingerprint = (bindings: string[][]) => createHash("sha256").update(JSON.stringify(bindings.map(b => [...b]).sort((a,b) => JSON.stringify(a) < JSON.stringify(b) ? -1 : JSON.stringify(a) > JSON.stringify(b) ? 1 : 0))).digest("hex"); + +/** Read current binding authority in the safety transaction. No network call, renewal or write. */ +export async function readIdleIncomingWakeProof(tx: Pick, input: { + capabilities?: VerifiedIdleCapabilities; ownerId: string; receiverOwner: object; +}): Promise { + const capability = input.capabilities?.incomingWake; + if (!capability) return; + if (capability.ownerId !== input.ownerId) throw new Error("Incoming wake owner changed"); + const proof: IdleIncomingWakeProof = { capability, connectionIds: [], endpointIds: [], requiredContinuityConnectionIds: [] }; + if (!capability.routes.includes("github-broker-events-v1")) return proof; + const broker = capability.githubBroker; + const config = paperclipCloudConnectorConfigFromEnv(); + if (!broker || config?.instanceId !== broker.instanceId || !hasConnectionEventWakeReceiver(input.receiverOwner)) + throw new Error("Incoming GitHub receiver is not bound"); + const rows = await tx.select({ grant: connectionGrants, connection: toolConnections }).from(connectionGrants) + .innerJoin(toolConnections, and(eq(toolConnections.id, connectionGrants.connectionId), eq(toolConnections.companyId, connectionGrants.companyId))) + .where(and(eq(connectionGrants.status, "active"), eq(toolConnections.status, "active"), eq(toolConnections.enabled, true), + sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`, + sql`exists (select 1 from companies where id=${connectionGrants.companyId} and status <> 'archived')`)); + if (rows.length > 1000) throw new Error("Too many GitHub bindings to inspect"); + for (const { grant, connection } of rows) { + const github = grant.providerTenant?.github; + if (!github || !Array.isArray(github.installationIds) || !github.installationIds.length || github.bindingSync || github.continuityRetry + || github.installationIds.some(id => typeof id !== "string" || !/^[0-9]+$/.test(id))) throw new Error("Unsettled GitHub authority"); + proof.requiredContinuityConnectionIds.push(connection.id); + } + const bindings = githubConnectionEventBindings(rows); + if (bindings.length !== rows.reduce((n,r) => n + r.grant.providerTenant!.github!.installationIds.length, 0)) throw new Error("Incomplete GitHub binding"); + const tuples = bindings.map(b => [b.id,b.companyId,b.subject,"github.code",b.installationId,b.connectionId,b.grantId]); + const endpoints = await tx.select({ endpoint: chatEndpoints, connection: toolConnections }).from(chatEndpoints) + .innerJoin(toolConnections, and(eq(toolConnections.id,chatEndpoints.connectionId),eq(toolConnections.companyId,chatEndpoints.companyId))) + .where(and(eq(chatEndpoints.provider,"github"),eq(chatEndpoints.status,"active"),eq(toolConnections.status,"active"),eq(toolConnections.enabled,true), + eq(toolConnections.transport,"chat_sdk"),eq(toolConnections.connectionPurpose,"channel"), + sql`exists (select 1 from companies where id=${chatEndpoints.companyId} and status <> 'archived')`)); + for (const { endpoint, connection } of endpoints) { + const registration = endpoint.setup.github?.cloudRegistrationId; + if (!registration || !/^[A-Za-z0-9_-]{43}$/.test(registration)) continue; + const registrations = await tx.select({ userId: chatGitHubRegistrations.userId }).from(chatGitHubRegistrations) + .where(and(eq(chatGitHubRegistrations.endpointId, endpoint.id), eq(chatGitHubRegistrations.companyId, endpoint.companyId), + eq(chatGitHubRegistrations.status, "completed"), sql`${chatGitHubRegistrations.handoff}->>'cloudId' = ${registration}`)).limit(2); + if (registrations.length !== 1 || !registrations[0]!.userId) throw new Error("GitHub registration authority changed"); + tuples.push([`github-app:${registration}`,endpoint.companyId,registrations[0]!.userId,"github.bot",`gateway:${registration}`,registration,registration]); + proof.connectionIds.push(connection.id); proof.endpointIds.push(endpoint.id); + } + if (incomingWakeFingerprint(tuples) !== broker.bindingsFingerprint) throw new Error("GitHub wake bindings changed"); + return proof; +} + +/** Narrowly substitute proven rows while leaving every other ingress predicate intact. */ +export function incomingWakeWorkCheck(query: string, proof?: IdleIncomingWakeProof, continuityConnectionIds: readonly string[] = []): SQL { + if (proof?.capability.routes.includes("tenant-api-http-v1") && (query.startsWith("SELECT 1 FROM agent_api_keys ") || query.startsWith("SELECT 1 FROM board_api_keys "))) return sql`false`; + const endpointPrefix = "SELECT 1 FROM chat_endpoints endpoint WHERE "; + const connectionPrefix = "SELECT 1 FROM tool_connections c WHERE "; + const connectionIds = [...proof?.connectionIds ?? [], ...proof?.requiredContinuityConnectionIds.filter(id => continuityConnectionIds.includes(id)) ?? []]; + if (proof?.endpointIds.length && query.startsWith(endpointPrefix)) return sql`EXISTS (SELECT 1 FROM chat_endpoints endpoint WHERE endpoint.id NOT IN (${sql.join(proof.endpointIds.map(id => sql`${id}::uuid`),sql`, `)}) AND (${sql.raw(query.slice(endpointPrefix.length))}))`; + if (connectionIds.length && query.startsWith(connectionPrefix)) return sql`EXISTS (SELECT 1 FROM tool_connections c WHERE c.id NOT IN (${sql.join(connectionIds.map(id => sql`${id}::uuid`),sql`, `)}) AND (${sql.raw(query.slice(connectionPrefix.length))}))`; + return sql`EXISTS (${sql.raw(query)})`; +} diff --git a/server/src/services/idle-local-work.ts b/server/src/services/idle-local-work.ts index 95b6549ebbc..0cd25aae858 100644 --- a/server/src/services/idle-local-work.ts +++ b/server/src/services/idle-local-work.ts @@ -1,3 +1,4 @@ +import { idleApiActivity } from "./idle-api-activity.js"; import { promises as fs } from "node:fs"; import path from "node:path"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; @@ -65,7 +66,7 @@ export async function readIdleLocalWork(options: { backupCheckpoint?: boolean } if (!startupComplete || !ingressTracked) return "unknown"; // This exemption belongs only to a scan which will finish a new backup // under the same admission hold before it can return an empty report. - if ((scheduledBackups && !options.backupCheckpoint) || idleWorkSnapshot().active !== 0) return "present"; + if (idleApiActivity.active() || (scheduledBackups && !options.backupCheckpoint) || idleWorkSnapshot().active !== 0) return "present"; try { const local = [...localWorkProbes].map(probe => probe()); if (local.includes("unknown")) return "unknown"; diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index c0d58a64bdd..59f08256b64 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -1,3 +1,5 @@ +import { readIdleIncomingWakeProof, incomingWakeWorkCheck, type IdleIncomingWakeProof } from "./idle-incoming-wake.js"; +import type { VerifiedIdleCapabilities, IdleIncomingWakeCapability } from "./idle-incoming-wake-capability.js"; import { idleWorkSnapshot } from "./task-admission.js"; import { readIdleLocalWork, type IdleLocalWork } from "./idle-local-work.js"; import { readIdleRoutineSchedule, type ScheduledWake } from "./idle-scheduled-work.js"; @@ -20,9 +22,16 @@ export interface IdleSleepDrainStatus { } export interface IdleSleepSafety { - version: 1 | 2; + version: 1 | 2 | 3; backgroundWork: "none" | "present" | "unknown"; scheduledWake?: ScheduledWake; + incomingWake?: IdleIncomingWakeCapability; +} + +export interface IdleSleepSafetyOptions { + incomingWake?: VerifiedIdleCapabilities; + deadlineCapability?: VerifiedIdleCapabilities; + collectDiagnostics?: true; } export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) => Promise<{ @@ -298,6 +307,7 @@ export async function readIdleSleepSafety( prepareBackup?: () => Promise, scheduledRoutines = false, scheduledIssueMonitors = false, + options: IdleSleepSafetyOptions = {}, ): Promise { const unknown: IdleSleepSafety = { version: 1, backgroundWork: "unknown" }; const before = getDrainStatus(); @@ -316,12 +326,17 @@ export async function readIdleSleepSafety( quietPlugins = pluginWork.pluginIds; } let scheduledWake: ScheduledWake | undefined; + let incomingProof: IdleIncomingWakeProof | undefined; const blocked = await db.transaction(async (tx) => { await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + if (options.incomingWake?.incomingWake) { + if (!ownerId || ownerId !== before.ownerId) return undefined; + incomingProof = await readIdleIncomingWakeProof(tx, { capabilities: options.incomingWake, ownerId, receiverOwner: db }); + } const checks = WORK_CHECKS.filter(query => (!scheduledRoutines || query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`) && (!scheduledIssueMonitors || query !== `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`)) - .map((query) => sql`EXISTS (${sql.raw(query)})`); + .map((query) => incomingWakeWorkCheck(query, incomingProof)); checks.push(sql`EXISTS (SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id WHERE ${companyHoldMismatch()})`); checks.push(sql`EXISTS (SELECT 1 FROM heartbeat_runs WHERE ${pendingBudgetRunCancellationPredicate()})`); @@ -366,6 +381,12 @@ export async function readIdleSleepSafety( const localAfter = idleWorkSnapshot(); if (!sameQuietHold(before, after, now()) || localAfter.active !== 0 || expectedGeneration !== localAfter.generation) return unknown; + if (local === "none" && incomingProof) { + // Until the negotiated deadline catalog owns the combined proof, retain + // legacy scheduled obligations rather than silently omit their alarm. + if (scheduledWake) return unknown; + return { version: 3, backgroundWork: "none", incomingWake: incomingProof.capability }; + } if (local === "none" && scheduledWake) { if (Date.parse(scheduledWake.nextWakeAt) <= now()) return { version: 1, backgroundWork: "present" }; return { version: 2, backgroundWork: "none", scheduledWake };