diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index 71bd5a549e2..30622581e24 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -120,6 +120,15 @@ block sleep even when their watched issue is complete and no review has started. Missing migrations, malformed configuration and database errors produce unknown without exposing SQL or tenant data. +Finished login-session history permits sleep only after its promotion claim is +cleared and any retained provider reference has a matching released or expired +lease with successful cleanup and a release timestamp in the same company and +environment. Missing cleanup evidence, `cleanup_pending`, and live login flows +remain blockers. A finished coalesced wake is history only when its exact +company/agent's linked run has finished without retry or accounting debt. +Image-setup history still blocks: promotion can succeed before teardown is +confirmed, so a terminal label does not prove that its provider resource is gone. + Every enabled plugin blocks sleep. A version label does not prove that arbitrary worker code has no background activity. No plugin approval or configuration bypass is supported. Scheduled and externally triggered work needs a durable diff --git a/server/src/__tests__/idle-sleep-safety.test.ts b/server/src/__tests__/idle-sleep-safety.test.ts index 66a023dd9b9..718d33ce534 100644 --- a/server/src/__tests__/idle-sleep-safety.test.ts +++ b/server/src/__tests__/idle-sleep-safety.test.ts @@ -178,7 +178,7 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp const companyId = randomUUID(); const agentId = randomUUID(); const runId = randomUUID(); - await db.insert(companies).values({ id: companyId, name: "Idle test", issuePrefix: "IDLE" }); + await db.insert(companies).values({ id: companyId, name: "Idle test", issuePrefix: `T${companyId.slice(0, 6).toUpperCase()}` }); await db.insert(agents).values({ id: agentId, companyId, name: "On-demand agent", role: "engineer", status: "idle", adapterType: "process" }); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" }); return { companyId, agentId, runId }; @@ -246,6 +246,44 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp expect(await read()).toEqual(present); }); + it("allows consumed coalesced wakes only after their linked run finishes", async () => { + const { companyId, agentId, runId } = await seed(); + await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, runId)); + await db.insert(agentWakeupRequests).values({ + companyId, agentId, source: "assignment", status: "coalesced", runId, finishedAt: new Date(now), + }); + expect(await read()).toEqual(present); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(now) }) + .where(eq(heartbeatRuns.id, runId)); + expect(await read()).toEqual(none); + }); + + it.each(["unfinished", "missing_run", "wrong_company", "wrong_agent", "unfinished_run", "retry", "accounting"])( + "keeps a coalesced wake awake with %s evidence", async kind => { + const { companyId, agentId, runId } = await seed(); + await db.update(heartbeatRuns).set({ finishedAt: new Date(now) }).where(eq(heartbeatRuns.id, runId)); + let linkedRunId = runId; + if (kind === "missing_run") linkedRunId = randomUUID(); + if (kind === "wrong_company") { + linkedRunId = (await seed()).runId; + await db.update(heartbeatRuns).set({ finishedAt: new Date(now) }).where(eq(heartbeatRuns.id, linkedRunId)); + } + if (kind === "wrong_agent") { + const otherAgentId = randomUUID(); + await db.insert(agents).values({ id: otherAgentId, companyId, name: "Other agent", role: "engineer", status: "idle", adapterType: "process" }); + await db.update(heartbeatRuns).set({ agentId: otherAgentId }).where(eq(heartbeatRuns.id, runId)); + } + if (kind === "unfinished_run") await db.update(heartbeatRuns).set({ finishedAt: null }).where(eq(heartbeatRuns.id, runId)); + if (kind === "retry") await db.update(heartbeatRuns).set({ scheduledRetryAt: new Date(now + 1000) }).where(eq(heartbeatRuns.id, runId)); + if (kind === "accounting") await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, runId)); + await db.insert(agentWakeupRequests).values({ + companyId, agentId, source: "assignment", status: "coalesced", runId: linkedRunId, + finishedAt: kind === "unfinished" ? null : new Date(now), + }); + expect(await read()).toEqual(present); + }, + ); + it("blocks an active routine without waiting for its next due time", async () => { const { companyId } = await seed(); await db.insert(routines).values({ companyId, title: "Tomorrow", status: "active" }); @@ -268,6 +306,95 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp expect(await read()).toEqual(present); }); + async function seedFinishedLogin() { + const { companyId } = await seed(); + const [environment] = await db.insert(environments).values({ name: "Finished login fixture" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ + companyId, environmentId: environment!.id, providerLeaseId: "fixture-login-resource", + status: "released", cleanupStatus: "success", releasedAt: new Date(now), + }).returning(); + const [session] = await db.insert(adapterAuthSessions).values({ + companyId, environmentId: environment!.id, adapterType: "codex_local", + startedByUserId: "fixture-user", publicSessionId: "finished-fixture-session", + status: "authenticated", finishedAt: new Date(now), providerLeaseId: lease!.providerLeaseId, + }).returning(); + return { companyId, environment: environment!, lease: lease!, session: session! }; + } + + it.each(["authenticated", "completed", "failed", "timed_out", "cancelled"] as const)( + "permits finished %s login history after confirmed provider cleanup", async status => { + const { session } = await seedFinishedLogin(); + await db.update(adapterAuthSessions).set({ status }).where(eq(adapterAuthSessions.id, session.id)); + expect(await read()).toEqual(none); + }, + ); + + it.each(["canonical", "uppercase"])("accepts a %s cleanup reference to the internal lease id", async format => { + const { session, lease } = await seedFinishedLogin(); + await db.update(adapterAuthSessions).set({ providerLeaseId: format === "uppercase" ? lease.id.toUpperCase() : lease.id }) + .where(eq(adapterAuthSessions.id, session.id)); + expect(await read()).toEqual(none); + }); + + it.each(["fixture-provider-id", "00000000-0000-0000-0000-00000000000z", randomUUID()])( + "accepts cleaned external provider reference %s without an unsafe UUID cast", async providerLeaseId => { + const { session, lease } = await seedFinishedLogin(); + await db.update(environmentLeases).set({ providerLeaseId }).where(eq(environmentLeases.id, lease.id)); + await db.update(adapterAuthSessions).set({ providerLeaseId }).where(eq(adapterAuthSessions.id, session.id)); + expect(await read()).toEqual(none); + }, + ); + + it("keeps a UUID-shaped external reference awake when an internal lease conflicts", async () => { + const { companyId, environment, session, lease } = await seedFinishedLogin(); + const conflictingId = randomUUID(); + await db.update(environmentLeases).set({ providerLeaseId: conflictingId }).where(eq(environmentLeases.id, lease.id)); + await db.update(adapterAuthSessions).set({ providerLeaseId: conflictingId }).where(eq(adapterAuthSessions.id, session.id)); + await db.insert(environmentLeases).values({ + id: conflictingId, companyId, environmentId: environment.id, + status: "released", cleanupStatus: "failed", releasedAt: new Date(now), + }); + expect(await read()).toEqual(present); + await db.update(environmentLeases).set({ cleanupStatus: "success" }).where(eq(environmentLeases.id, conflictingId)); + expect(await read()).toEqual(none); + }); + + it("permits terminal login history without a provider resource", async () => { + const { session } = await seedFinishedLogin(); + await db.update(adapterAuthSessions).set({ providerLeaseId: null }).where(eq(adapterAuthSessions.id, session.id)); + expect(await read()).toEqual(none); + }); + + it.each(["cleanup_pending", "starting", "promoting", "awaiting_code", "submitting", "stored"] as const)( + "retains %s login work even with an old successful cleanup receipt", async status => { + const { session } = await seedFinishedLogin(); + await db.update(adapterAuthSessions).set({ status }).where(eq(adapterAuthSessions.id, session.id)); + expect(await read()).toEqual(present); + }, + ); + + it.each(["unfinished", "promotion_claim", "missing_lease", "wrong_company", "wrong_environment", "cleanup_failed", "cleanup_unknown", "unreleased", "conflicting_lease"])( + "retains terminal login history with %s cleanup evidence", async kind => { + const { companyId, session, lease, environment } = await seedFinishedLogin(); + if (kind === "unfinished") await db.update(adapterAuthSessions).set({ finishedAt: null }).where(eq(adapterAuthSessions.id, session.id)); + if (kind === "promotion_claim") await db.update(adapterAuthSessions).set({ promotionExpiresAt: new Date(now + 1000) }).where(eq(adapterAuthSessions.id, session.id)); + if (kind === "missing_lease") await db.delete(environmentLeases).where(eq(environmentLeases.id, lease.id)); + if (kind === "wrong_company") await db.update(environmentLeases).set({ companyId: (await seed()).companyId }).where(eq(environmentLeases.id, lease.id)); + if (kind === "wrong_environment") { + const [other] = await db.insert(environments).values({ name: "Other environment", driver: "sandbox" }).returning(); + await db.update(environmentLeases).set({ environmentId: other!.id }).where(eq(environmentLeases.id, lease.id)); + } + if (kind === "cleanup_failed") await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, lease.id)); + if (kind === "cleanup_unknown") await db.update(environmentLeases).set({ cleanupStatus: null }).where(eq(environmentLeases.id, lease.id)); + if (kind === "unreleased") await db.update(environmentLeases).set({ releasedAt: null }).where(eq(environmentLeases.id, lease.id)); + if (kind === "conflicting_lease") await db.insert(environmentLeases).values({ + companyId, environmentId: environment.id, providerLeaseId: lease.providerLeaseId, + status: "released", cleanupStatus: "failed", releasedAt: new Date(now), + }); + expect(await read()).toEqual(present); + }, + ); + it("reports a pending secret proposal with future expiry", async () => { const { companyId, agentId, runId } = await seed(); await db.insert(companySecretProposals).values({ diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index a8c7a80e13b..4f2059f3bb4 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -45,8 +45,16 @@ const WORK_CHECKS = [ `SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`, `SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL AND (expires_at IS NULL OR expires_at > now())`, - `SELECT 1 FROM agent_wakeup_requests WHERE - status NOT IN ('completed', 'failed', 'cancelled', 'skipped', 'timed_out')`, + // Coalescing consumes a request into another run. Retain it as work until + // both the request and that exact company/agent's run have finished. + `SELECT 1 FROM agent_wakeup_requests w WHERE + w.status NOT IN ('completed', 'failed', 'cancelled', 'skipped', 'timed_out', 'coalesced') + OR (w.status = 'coalesced' AND (w.finished_at IS NULL OR NOT EXISTS ( + SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_id + AND r.company_id = w.company_id AND r.agent_id = w.agent_id + AND r.status IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted') + AND r.finished_at IS NOT NULL AND r.scheduled_retry_at IS NULL + AND r.cost_accounting_pending = false)))`, `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`, // A completed issue can still need its first watchdog review, including a // retry after immediate evaluation failed before creating a review/run. @@ -62,6 +70,29 @@ const WORK_CHECKS = [ `SELECT 1 FROM issue_recovery_actions WHERE status NOT IN ('resolved', 'cancelled')`, `SELECT 1 FROM status_cards WHERE archived_at IS NULL`, `SELECT 1 FROM external_objects WHERE NOT is_terminal OR next_refresh_at IS NOT NULL`, + // Login history keeps its provider reference after teardown. A terminal + // label alone is insufficient: require a finished, unclaimed session and + // positive cleanup evidence for its resource in the same company/environment. + // Cast only UUID-shaped session references, leaving both lease indexes usable. + `SELECT 1 FROM adapter_auth_sessions s WHERE + s.status NOT IN ('authenticated', 'completed', 'failed', 'timed_out', 'cancelled') + OR s.finished_at IS NULL OR s.promotion_expires_at IS NOT NULL + OR (s.provider_lease_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM environment_leases l WHERE l.company_id = s.company_id + AND l.environment_id = s.environment_id + AND (l.provider_lease_id = s.provider_lease_id OR l.id = CASE + WHEN s.provider_lease_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + THEN s.provider_lease_id::uuid END) + AND l.status IN ('released', 'expired') AND l.cleanup_status = 'success' + AND l.released_at IS NOT NULL)) + OR (s.provider_lease_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM environment_leases l WHERE l.company_id = s.company_id + AND l.environment_id = s.environment_id + AND (l.provider_lease_id = s.provider_lease_id OR l.id = CASE + WHEN s.provider_lease_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + THEN s.provider_lease_id::uuid END) + AND (l.status IN ('released', 'expired') AND l.cleanup_status = 'success' + AND l.released_at IS NOT NULL) IS NOT TRUE))`, // These less common work sources fail closed on any retained state. Their // terminal-state exceptions can be added with tests for the owning service. ...[ @@ -73,7 +104,7 @@ const WORK_CHECKS = [ "native_run_finalizations", "company_transfer_runs", "decisions", "decision_effect_executions", "decision_archive_notification_outbox", "browser_use_sessions", "browser_use_runs", "browser_use_browsers", "environment_custom_image_setup_sessions", "feedback_exports", - "adapter_auth_sessions", "company_secret_proposals", "execution_workspaces", + "company_secret_proposals", "execution_workspaces", "mcp_oauth_grants", "mcp_mutation_receipts", "mcp_event_subscriptions", "mcp_event_deliveries", "mcp_attachment_uploads", "dot_agent_bindings", "dot_runner_assignments", "dot_runner_operations", "dot_mailbox_items",