From 1423d745e84e1f2362b51150d466dc112e287cf4 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:45:10 -0600 Subject: [PATCH] feat(server): expose owned sleep blocker diagnostics (#18632) Reviewed and verified at 26e5a22b3c23ccd4e6ad101178c5328264f7bea7. Co-Authored-By: Paperclip --- doc/idle-sleep-safety.md | 21 +- .../2026-11-02-sleep-blocker-inventory.md | 164 ++++++++++ .../src/drivers/acpx/runtime-host.test.ts | 11 +- server/src/__tests__/idle-local-work.test.ts | 29 ++ .../src/__tests__/idle-scheduled-work.test.ts | 2 +- .../idle-sleep-diagnostics-postgres.test.ts | 130 ++++++++ .../__tests__/idle-sleep-diagnostics.test.ts | 217 +++++++++++++ .../src/__tests__/idle-sleep-safety.test.ts | 10 +- .../instance-settings-routes.test.ts | 22 ++ .../issue-recovery-assignments.test.ts | 6 +- server/src/routes/instance-settings.ts | 5 +- server/src/services/idle-issue-schedule.ts | 3 +- server/src/services/idle-local-work.ts | 52 +++- server/src/services/idle-scheduled-work.ts | 3 +- server/src/services/idle-sleep-diagnostics.ts | 46 +++ server/src/services/idle-sleep-safety.ts | 290 +++++++++++------- 16 files changed, 879 insertions(+), 132 deletions(-) create mode 100644 doc/plans/2026-11-02-sleep-blocker-inventory.md create mode 100644 server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts create mode 100644 server/src/__tests__/idle-sleep-diagnostics.test.ts create mode 100644 server/src/services/idle-sleep-diagnostics.ts diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index 8334e3b1685..1cf6fe81272 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -5,6 +5,22 @@ instance-wide `idleSleepSafety` report for instance administrators and signed Cloud control callers. Ordinary task drains keep their existing behavior. Company members and agents cannot read this report. +Instance administrators and signed Cloud control callers may add +`idleSleepDiagnostics=1` to an owned safety request for fixed-key blocker +diagnostics. This explicit one-shot read evaluates all current predicates in +the same bounded snapshot, and can reconnect/wake the database. Do not poll it +during a suspension observation window. The normal report and its early local +and plugin short circuits are unchanged when the option is absent. + +The optional `diagnostics` field reports `none`, `present`, `unknown`, or +`not_checked` per reason. `complete` describes applicable read-inspection +coverage, never stop authorization. Diagnostics do not initiate backup dumps +or unsafe plugin RPCs. Skipped checkpoints remain `not_checked`; unknown or +skipped checks never grant a safety `none`. A changed hold or tracked-work +generation invalidates the observation. Only fixed keys and states are +returned, with no tenant records or error details. See the +[complete reason and traffic inventory](plans/2026-11-02-sleep-blocker-inventory.md). + ## Hosting protocol 1. Hold external admission at the hosting layer. Queue user requests for wake or @@ -413,8 +429,9 @@ the same termination and run checks above. Missing cleanup evidence, `cleanup_pending`, and live login flows remain blockers. A finished coalesced wake is history only when its exact company/agent's linked run has finished without retry or accounting debt. -Image-setup history still blocks: promotion can succeed before teardown is -confirmed, so a terminal label does not prove that its provider resource is gone. +Image-setup history requires a known terminal state, finish timestamp, cleared +connection secret and exact host-owned resource cleanup verification. Promotion +can succeed before teardown is confirmed, so a terminal label alone still blocks. Completed question-answer deliveries permit sleep only after acknowledgement and completion of their exact target run in the same company, without retry or diff --git a/doc/plans/2026-11-02-sleep-blocker-inventory.md b/doc/plans/2026-11-02-sleep-blocker-inventory.md new file mode 100644 index 00000000000..e9b1de4ce71 --- /dev/null +++ b/doc/plans/2026-11-02-sleep-blocker-inventory.md @@ -0,0 +1,164 @@ +# Sleep blocker inventory and one-shot diagnostics + +Date: 2026-11-02. Source baseline: `0748b9006273dae7ef8c303905ca0ffa0221a708`. + +This inventory describes the current application policy. It preserves every blocker and retained record. It does not qualify Cloud ingress, external alarms, resource shutdown, backup restoration, database suspension, or host suspension. The policy remains in [idle-sleep-safety.ts](../../server/src/services/idle-sleep-safety.ts); domain owners retain completion, authorization, company linkage, retries, and cleanup. + +## Operator diagnostic + +Hold hosting admission and acquire an owned idle drain using the existing [hosting protocol](../idle-sleep-safety.md). An instance administrator or signed Cloud control caller can make one explicit request: + +```text +GET /api/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&ownerId= +``` + +The ordinary response remains unchanged unless diagnostics are requested. A diagnostic adds `idleSleepSafety.diagnostics: { version: 1, complete, checks: [{ reason, state }] }`. Only fixed reason identifiers and states leave the inspector; it does not return company/plugin/row identifiers, values, counts, paths, SQL, credentials, or errors. Company members and agents cannot read this instance-wide report. Negotiated `scheduledWake=1/2` remains exclusive to verified Cloud control; managed checkpoint authority retains its existing restriction. + +States are `none`, `present`, `unknown`, and `not_checked`. `complete` covers applicable **read inspections** under the same current owned hold and generation. It is not a stop certificate. An unperformed backup checkpoint is explicitly `not_checked` even if all read inspection completed. Diagnostics never initiate backups to discover their outcome or force plugin RPCs past unsafe lifecycle fences. No unknown or skipped result can authorize safety `none`. + +The opt-in DB SELECT evaluates every existing predicate as a named boolean column, instead of the normal OR short circuit. It runs once in the existing read-only repeatable-read transaction with a five-second statement timeout. It can cost more than an ordinary report and can reconnect/wake the DB. Do not poll it or use it during a suspension observation window. Ownership loss, expired/changed hold, concurrent tracked work, malformed results/configuration, missing migrations, timeout, or unreadable local state remains unknown/incomplete. A failed PostgreSQL statement aborts that snapshot; no later out-of-transaction scan fabricates completeness. Already observed reasons are historical observations of this invalidated request, not current clearance. + +## Complete durable reason catalog + +At this baseline there are 36 explicit checks, 32 retained-table checks, and four appended checks: 72 durable predicates. Keys identify domain families independently of SQL position or status. A later completion/ingress/deadline improvement retains the family key; a new independent predicate requires its own key and inventory row. This count is a baseline observation, not a permanent protocol limit. + +For every row below, the diagnostic fixture is the named durable trigger and its negative/completion counterpart. Existing owner suites establish domain contracts; this PR's real PostgreSQL representative fixtures establish simultaneous reporting and the original eligibility behavior, not new exemptions for every domain. The reason-column test compiles the actual full policy query, checks key uniqueness, and evaluates its reporting contract. It is not substituted for real database fixtures or deployed proof. + +Supported writes remain company scoped in their owning services. Queue-producing writes use [transaction work signals](../DATABASE.md#transaction-aware-delivery-work-signals) and the [delivery coordinator](../../server/src/services/delivery-work-coordinator.ts): intent before write, notification only after outer commit, unresolved/ambiguous outcomes retained, startup reconstruction. A direct SQL fixture is setup, not evidence that an independent writer participates in those signals. The existing single-active-process writer contract does not cover independent SQL writers or extra replicas. + +Remedies below mean completing the owning protocol or supplying a separately tested follow-up proof. They never mean deleting history, revoking working credentials, pausing customer functionality, or clearing uncertain debt merely to achieve quiet. + +| Reason | Durable truth / owner and producers | Current pending versus passive proof; fixture trigger | Completion dependency / remedy | +| --- | --- | --- | --- | +| `agents.execution_or_timer` | [agents](../../packages/db/src/schema/agents.ts); [heartbeat/scheduling.ts](../../server/src/services/heartbeat/scheduling.ts). active / future. | Running agent, or enabled positive timer on an executable agent. Paused/terminated/pending approval timers do not require dispatch. Fixture: Enable a positive timer; disable it for the negative fixture. | Future-work alarm coverage; preserve the owning evidence and records. | +| `heartbeat_runs` | [heartbeat_runs](../../packages/db/src/schema/heartbeat_runs.ts); [heartbeat/run-completion.ts](../../server/src/services/heartbeat/run-completion.ts). active / recovery. | Nonterminal run, retry later than finish or missing finish, or pending accounting. Terminal consumed retries alone are history. Fixture: Queued run versus finished ordinary run, consumed retry, live retry child. | Domain settlement; preserve the owning evidence and records. | +| `agent_instruction_working_copies` | [agent_instruction_working_copies](../../packages/db/src/schema/agent_instruction_revisions.ts); [agent-instruction-working-copies.ts](../../server/src/services/agent-instruction-working-copies.ts). cleanup. | Requires known settled state, stopped process, no next attempt, no baseline or cleanup-pending receipt. Fixture: Stopped saved copy versus deferred cleanup. | Domain cleanup; preserve the owning evidence and records. | +| `sandbox_creation_operations` | [sandbox_creation_operations](../../packages/db/src/schema/sandbox_creation_operations.ts); [sandbox-creation-operations.ts](../../server/src/services/sandbox-creation-operations.ts). recovery. | Pending creation operation retains uncertain/provider work. Fixture: Pending operation versus settled operation. | Domain settlement; preserve the owning evidence and records. | +| `budget_reservations` | [budget_reservations](../../packages/db/src/schema/accounting.ts); [budget-reservations.ts](../../server/src/services/budget-reservations.ts). accounting. | Anything outside settled/released remains owed. Fixture: Unsettled reservation versus settled charge. | Domain accounting; preserve the owning evidence and records. | +| `budget_policies` | [budget_policies](../../packages/db/src/schema/budget_policies.ts); [budgets.ts](../../server/src/services/budgets.ts). accounting. | Enforcement generation exceeds delivered generation. Fixture: Committed enforcement version2/delivered1 versus delivered2. | Domain delivery; preserve the owning evidence and records. | +| `agents.lifecycle` | [agents](../../packages/db/src/schema/agents.ts); [../modules/agent-lifecycle/index.ts](../../server/src/modules/agent-lifecycle/index.ts). active / recovery. | Preparing/verifying/pausing/resuming/terminating/cleaning-up phases. Fixture: Pausing agent versus stable paused state. | Domain settlement; preserve the owning evidence and records. | +| `decision_invocations` | [decision_invocations](../../packages/db/src/schema/decision_models.ts); [decision-models.ts](../../server/src/services/decision-models.ts). active. | Running decision invocation. This existing predicate is narrower than a universal unknown-state guard. Fixture: Running invocation versus completed invocation. | Domain completion; preserve the owning evidence and records. | +| `prompt_injection_invocations` | [prompt_injection_invocations](../../packages/db/src/schema/prompt_injection_prevention.ts); [prompt-injection-prevention.ts](../../server/src/services/prompt-injection-prevention.ts). active. | Running prompt-injection invocation; current SQL checks running specifically. Fixture: Running invocation versus finished invocation. | Domain completion; preserve the owning evidence and records. | +| `agent_api_keys` | [agent_api_keys](../../packages/db/src/schema/agent_api_keys.ts); [agents.ts](../../server/src/services/agents.ts). ingress configuration. | Unrevoked machine credential can accept autonomous inbound work. Fixture: Saved live key versus revoked key. | Incoming-work proof; preserve the owning evidence and records. | +| `board_api_keys` | [board_api_keys](../../packages/db/src/schema/board_api_keys.ts); [board-auth.ts](../../server/src/services/board-auth.ts). ingress configuration. | Unrevoked key whose expiry is absent or future. Fixture: Live key versus revoked/expired key. | Incoming-work proof; preserve the owning evidence and records. | +| `agent_wakeup_requests` | [agent_wakeup_requests](../../packages/db/src/schema/agent_wakeup_requests.ts); [heartbeat/queue.ts](../../server/src/services/heartbeat/queue.ts). active / handoff. | Unsettled wake; coalesced wake additionally requires finished receipt and exact company/agent finished run. Fixture: Deferred wake versus exact completed/coalesced receipt. | Domain handoff; preserve the owning evidence and records. | +| `issues` | [issues](../../packages/db/src/schema/issues.ts); [issues.ts](../../server/src/services/issues.ts). active / passive / future. | Ordinary mode blocks open tasks; negotiated monitor capability replaces this predicate with owning issue schedule proof. Fixture: Backlog task ordinary versus negotiated passive mode. | Future-work proof; preserve the owning evidence and records. | +| `issue_watchdogs` | [issue_watchdogs](../../packages/db/src/schema/issue_watchdogs.ts); [task-watchdogs.ts](../../server/src/services/task-watchdogs.ts). future. | Every enabled watchdog, including completed issue before first review. Fixture: Enabled watchdog on a done task versus disabled watchdog. | Future-work alarm coverage; preserve the owning evidence and records. | +| `workspace_operations` | [workspace_operations](../../packages/db/src/schema/workspace_operations.ts); [workspace-operations.ts](../../server/src/services/workspace-operations.ts). active / cleanup. | Anything outside succeeded/failed/cancelled. Fixture: Unfinished operation versus succeeded workspace finalization. | Domain settlement; preserve the owning evidence and records. | +| `execution_workspaces` | [execution_workspaces](../../packages/db/src/schema/execution_workspaces.ts); [execution-workspaces.ts](../../server/src/services/execution-workspaces.ts). resource / cleanup. | Only known local directory/worktree provider, mode, strategy and state, inert config, no inherited runtime/reopen or due cleanup qualify. Fixture: Retained plain local directory versus runtime config or due cleanup. | Resource proof; preserve the owning evidence and records. | +| `environment_leases` | [environment_leases](../../packages/db/src/schema/environment_leases.ts); [environment-runtime.ts](../../server/src/services/environment-runtime.ts). resource / cleanup. | Released/expired, or exact failed-lease termination proof. Shared proof retains unfinished run/accounting/retry/export/cleanup debt. Fixture: Verified failed ephemeral lease versus missing/foreign resource receipt. | Resource proof; preserve the owning evidence and records. | +| `environment_custom_image_setup_sessions` | [environment_custom_image_setup_sessions](../../packages/db/src/schema/environment_custom_image_setup_sessions.ts); [environment-custom-images.ts](../../server/src/services/environment-custom-images.ts). resource / cleanup. | Known terminal setup, finish, cleared connection secret and exact host resource cleanup verification. Fixture: Verified terminal setup versus absent cleanup receipt. | Resource proof; preserve the owning evidence and records. | +| `chat_endpoints` | [chat_endpoints](../../packages/db/src/schema/chat_channels.ts); [chat-channels/endpoints.ts](../../server/src/services/chat-channels/endpoints.ts). ingress configuration. | Only exact unconfigured Slack provider-setup draft with matching disabled draft connection and no identity/activity/error/registration is passive. Fixture: Unconfigured Slack draft versus partially connected/active endpoint. | Incoming-work proof; preserve the owning evidence and records. | +| `routines` | [routines](../../packages/db/src/schema/routines.ts); [routines.ts](../../server/src/services/routines.ts). future. | Anything outside paused/archived ordinary mode; negotiated routine reader validates active future triggers instead. Fixture: Active valid future trigger ordinary versus negotiated proof. | Future-work proof; preserve the owning evidence and records. | +| `routine_runs` | [routine_runs](../../packages/db/src/schema/routines.ts); [routines.ts](../../server/src/services/routines.ts). active / handoff. | Anything outside completed/failed/skipped/cancelled. Fixture: Pending handoff versus completed run. | Domain settlement; preserve the owning evidence and records. | +| `plugin_jobs` | [plugin_jobs](../../packages/db/src/schema/plugin_jobs.ts); [plugin-job-scheduler.ts](../../server/src/services/plugin-job-scheduler.ts). future. | Every nonpaused plugin job. Plugin worker quiet alone cannot transfer a future job obligation. Fixture: Scheduled job versus paused job. | Future-work alarm coverage; preserve the owning evidence and records. | +| `plugin_job_runs` | [plugin_job_runs](../../packages/db/src/schema/plugin_jobs.ts); [plugin-job-scheduler.ts](../../server/src/services/plugin-job-scheduler.ts). active. | Anything outside succeeded/failed/cancelled. Fixture: Running job versus succeeded run. | Domain settlement; preserve the owning evidence and records. | +| `plugin_webhook_deliveries` | [plugin_webhook_deliveries](../../packages/db/src/schema/plugin_webhooks.ts); [../routes/plugins.ts](../../server/src/routes/plugins.ts). delivery. | Anything outside succeeded/failed. Fixture: Pending webhook versus terminal delivery. | Domain settlement; preserve the owning evidence and records. | +| `tool_connections` | [tool_connections](../../packages/db/src/schema/tool_access.ts); [tool-access.ts](../../server/src/services/tool-access.ts). configuration / resource. | Archived, disabled draft, active saved AI runtime-auth, or outbound remote tool MCP without foreign/GitHub grant binding can be passive. Other transports remain blocked. Fixture: Saved outbound MCP/AI versus local stdio/channel/webhook binding. | Incoming-work proof; preserve the owning evidence and records. | +| `issue_recovery_actions` | [issue_recovery_actions](../../packages/db/src/schema/issue_recovery_actions.ts); [recovery/index.ts](../../server/src/services/recovery/index.ts). recovery / passive. | Unresolved actions unless existing exact board escalation proof establishes no automatic retry/live execution/monitor/timeout. Fixture: Known board-owned passive escalation versus automated recovery. | Domain recovery; preserve the owning evidence and records. | +| `status_cards` | [status_cards](../../packages/db/src/schema/status_cards.ts); [status-cards.ts](../../server/src/services/status-cards.ts). future / active. | Every unarchived status card. Fixture: Unarchived card versus archived card. | Future-work/domain proof; preserve the owning evidence and records. | +| `external_objects` | [external_objects](../../packages/db/src/schema/external_objects.ts); [external-objects.ts](../../server/src/services/external-objects.ts). future / active. | Nonterminal external object or any next-refresh timestamp. Fixture: Finished object without refresh versus saved next refresh. | Future-work alarm coverage; preserve the owning evidence and records. | +| `adapter_auth_sessions` | [adapter_auth_sessions](../../packages/db/src/schema/adapter_auth_sessions.ts); [setup-token-session.ts](../../server/src/services/setup-token-session.ts). resource / cleanup. | Known terminal session, finish, cleared promotion claim and exact same-company/environment released or verified resource cleanup. Fixture: Finished no-resource login versus live login/unclean lease. | Resource proof; preserve the owning evidence and records. | +| `issue_question_response_deliveries` | [issue_question_response_deliveries](../../packages/db/src/schema/issue_question_response_deliveries.ts); [question-response-delivery.ts](../../server/src/services/question-response-delivery.ts). delivery / handoff. | Acknowledgement and exact target-run or fallback-wake completion; coalesced fallback needs exact company/agent run. Failed no-target history is passive. Fixture: Acknowledged handoff versus missing acknowledgement/unfinished target. | Domain handoff; preserve the owning evidence and records. | +| `connection_event_deliveries` | [connection_event_deliveries](../../packages/db/src/schema/connection_event_deliveries.ts); [github-connection-events.ts](../../server/src/services/github-connection-events.ts). delivery / history. | Processed with processing timestamp is idempotency history; otherwise work remains. Downstream work is independent. Fixture: Processed stamped receipt versus received unstamped receipt. | Domain delivery; preserve the owning evidence and records. | +| `connection_intent_deliveries` | [connection_intent_deliveries](../../packages/db/src/schema/connection_intent_deliveries.ts); [connection-intent-delivery.ts](../../server/src/services/connection-intent-delivery.ts). delivery / passive. | Delivered with exact finished wakes, or existing response-wait proof. Retired parked outbox stays durable. Fixture: Delivered wake-finished history versus pending downstream wake. | Domain handoff; preserve the owning evidence and records. | +| `feedback_exports` | [feedback_exports](../../packages/db/src/schema/feedback_exports.ts); [feedback.ts](../../server/src/services/feedback.ts). delivery / history. | Local-only, or sent with completion timestamp and no failure. Pending/failed/unfamiliar remains work. Fixture: Private local feedback or stamped sent versus inconsistent sent. | Domain export; preserve the owning evidence and records. | +| `project_skill_sources` | [project_skill_sources](../../packages/db/src/schema/company_skill_sources.ts); [skill-source-sync.ts](../../server/src/services/skill-source-sync.ts). configuration / maintenance. | Active company/project binding with no source ID. Optional package conversion is separate maintenance. Fixture: Active legacy binding versus converted binding/inactive company. | Domain source sync; preserve the owning evidence and records. | +| `company_skill_sources` | [company_skill_sources](../../packages/db/src/schema/company_skill_sources.ts); [skill-source-sync.ts](../../server/src/services/skill-source-sync.ts). future. | Active-company enabled automatic source updates with company scope or active project binding. Fixture: Enabled automatic source versus disabled/manual source. | Future-work coverage; preserve the owning evidence and records. | +| `native_run_finalizations` | [native_run_finalizations](../../packages/db/src/schema/native_run_finalizations.ts); [native-runtime/native-finalization-reconciler.ts](../../server/src/services/native-runtime/native-finalization-reconciler.ts). finalization / handoff. | Exact committed run/result/assessment/applied decision, finished accounting/retry, succeeded workspace finalize, delivered effects, no leases/attempt/recovery/failure. Fixture: Fully committed finalization versus missing/foreign dependency or pending effect. | Domain finalization; preserve the owning evidence and records. | +| `agents.company_hold` | [agents](../../packages/db/src/schema/agents.ts); [../modules/agent-lifecycle/index.ts](../../server/src/modules/agent-lifecycle/index.ts). recovery. | Committed company pause/archive and inherited agent hold mismatch. Fixture: Paused company with missing hold versus matching stable projection. | Domain lifecycle; preserve the owning evidence and records. | +| `heartbeat_runs.budget_cancellation` | [heartbeat_runs](../../packages/db/src/schema/heartbeat_runs.ts); [heartbeat/budget-cancellation.ts](../../server/src/services/heartbeat/budget-cancellation.ts). accounting / Stop. | Existing pending-budget-cancellation predicate; terminal run does not erase owed provider Stop. Fixture: Committed budget Stop versus acknowledged cancellation. | Domain accounting; preserve the owning evidence and records. | +| `native_response_presentation` | linked domain rows; [native-runtime/native-response-presentation-query.ts](../../server/src/services/native-runtime/native-response-presentation-query.ts). delivery. | Existing presentation candidate query over exact native finalization/decision/result; terminal run can still owe visible reply. Fixture: Committed reply without presentation versus settled presentation. | Domain presentation; preserve the owning evidence and records. | +| `plugins` | [plugins](../../packages/db/src/schema/plugins.ts); [plugin-worker-manager.ts](../../server/src/services/plugin-worker-manager.ts). runtime resource. | Every nondisabled plugin except exact current workers that returned an owned idle-drain proof. Startup/replacement/old SDK/crash/active route remain conservative. Fixture: Enabled plugin without proof versus exact quiet-worker proof. | Plugin lifecycle/ingress/deadline proof; preserve the owning evidence and records. | + +### Retained-table predicates + +Every row below currently blocks on **any retained row**, including completed-looking history. A terminal label in an owner suite is not a sleep exemption. Its trigger fixture is a retained schema-valid row from that owner suite; a no-row fixture is the current negative. New representative PostgreSQL fixtures explicitly exercise processed chat and completed transfer history alongside pending work, retaining all rows. The listed owner fixtures are construction/recovery references; running them alone is not a claim of sleep acceptance. + +| Reason | Durable truth / owning producer | Current trigger / fixture reference | Required follow-up | +| --- | --- | --- | --- | +| `chat_deliveries` | [chat_deliveries](../../packages/db/src/schema/chat_channels.ts); [chat-channels/inbound/delivery-ledger.ts](../../server/src/services/chat-channels/inbound/delivery-ledger.ts) | Any retained row, with no terminal exception. [chat-delivery-work.test.ts](../../server/src/__tests__/chat-delivery-work.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `chat_publications` | [chat_publications](../../packages/db/src/schema/chat_channels.ts); [chat-channels/publications/worker.ts](../../server/src/services/chat-channels/publications/worker.ts) | Any retained row, with no terminal exception. [chat-delivery-work.test.ts](../../server/src/__tests__/chat-delivery-work.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `chat_actions` | [chat_actions](../../packages/db/src/schema/chat_channels.ts); [chat-channels/interactions/confirmation-actions.ts](../../server/src/services/chat-channels/interactions/confirmation-actions.ts) | Any retained row, with no terminal exception. [chat-action-work.test.ts](../../server/src/__tests__/chat-action-work.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `chat_completion_deliveries` | [chat_completion_deliveries](../../packages/db/src/schema/chat_completion_deliveries.ts); [chat-completion-delivery.ts](../../server/src/services/chat-completion-delivery.ts) | Any retained row, with no terminal exception. [chat-completion-delivery.test.ts](../../server/src/__tests__/chat-completion-delivery.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `tool_action_requests` | [tool_action_requests](../../packages/db/src/schema/tool_access.ts); [tool-action-review.ts](../../server/src/services/tool-action-review.ts) | Any retained row, with no terminal exception. [tool-review-queue-unsigned-request.test.ts](../../server/src/__tests__/tool-review-queue-unsigned-request.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `tool_action_deliveries` | [tool_action_deliveries](../../packages/db/src/schema/tool_action_deliveries.ts); [tool-action-delivery.ts](../../server/src/services/tool-action-delivery.ts) | Any retained row, with no terminal exception. [tool-review-queue-unsigned-request.test.ts](../../server/src/__tests__/tool-review-queue-unsigned-request.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `workspace_runtime_services` | [workspace_runtime_services](../../packages/db/src/schema/workspace_runtime_services.ts); [workspace-runtime.ts](../../server/src/services/workspace-runtime.ts) | Any retained row, with no terminal exception. [workspace-runtime.test.ts](../../server/src/__tests__/workspace-runtime.test.ts) supplies domain fixtures. | History/resource ingress: verify exact completion/authority/resource and dependent effects before narrowing. | +| `execution_workspace_runtime_leases` | [execution_workspace_runtime_leases](../../packages/db/src/schema/execution_workspace_runtime_leases.ts); [workspace-runtime-leases.ts](../../server/src/services/workspace-runtime-leases.ts) | Any retained row, with no terminal exception. [workspace-runtime-leases.test.ts](../../server/src/__tests__/workspace-runtime-leases.test.ts) supplies domain fixtures. | History/resource ingress: verify exact completion/authority/resource and dependent effects before narrowing. | +| `company_transfer_runs` | [company_transfer_runs](../../packages/db/src/schema/company_transfer_runs.ts); [company-transfer-runs.ts](../../server/src/services/company-transfer-runs.ts) | Any retained row, with no terminal exception. [company-transfer-runs.test.ts](../../server/src/__tests__/company-transfer-runs.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `decisions` | [decisions](../../packages/db/src/schema/decisions.ts); [decisions.ts](../../server/src/services/decisions.ts) | Any retained row, with no terminal exception. [decisions-service.test.ts](../../server/src/__tests__/decisions-service.test.ts) supplies domain fixtures. | History/deadline: verify exact completion/authority/resource and dependent effects before narrowing. | +| `decision_effect_executions` | [decision_effect_executions](../../packages/db/src/schema/decisions.ts); [decisions.ts](../../server/src/services/decisions.ts) | Any retained row, with no terminal exception. [decision-effect-recovery-work.test.ts](../../server/src/__tests__/decision-effect-recovery-work.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `decision_archive_notification_outbox` | [decision_archive_notification_outbox](../../packages/db/src/schema/decision_queues.ts); [decision-archive-delivery.ts](../../server/src/services/decision-archive-delivery.ts) | Any retained row, with no terminal exception. [decision-retention-service.test.ts](../../server/src/__tests__/decision-retention-service.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `browser_use_sessions` | [browser_use_sessions](../../packages/db/src/schema/browser_use.ts); [browser-use.ts](../../server/src/services/browser-use.ts) | Any retained row, with no terminal exception. [browser-use.test.ts](../../server/src/__tests__/browser-use.test.ts) supplies domain fixtures. | History/resource: verify exact completion/authority/resource and dependent effects before narrowing. | +| `browser_use_runs` | [browser_use_runs](../../packages/db/src/schema/browser_use.ts); [browser-use.ts](../../server/src/services/browser-use.ts) | Any retained row, with no terminal exception. [browser-use.test.ts](../../server/src/__tests__/browser-use.test.ts) supplies domain fixtures. | History/resource: verify exact completion/authority/resource and dependent effects before narrowing. | +| `browser_use_browsers` | [browser_use_browsers](../../packages/db/src/schema/browser_use.ts); [browser-use.ts](../../server/src/services/browser-use.ts) | Any retained row, with no terminal exception. [browser-use-work.test.ts](../../server/src/__tests__/browser-use-work.test.ts) supplies domain fixtures. | History/resource: verify exact completion/authority/resource and dependent effects before narrowing. | +| `company_secret_proposals` | [company_secret_proposals](../../packages/db/src/schema/company_secret_proposals.ts); [secret-proposals.ts](../../server/src/services/secret-proposals.ts) | Any retained row, with no terminal exception. [secret-proposals-routes.test.ts](../../server/src/__tests__/secret-proposals-routes.test.ts) supplies domain fixtures. | History/deadline: verify exact completion/authority/resource and dependent effects before narrowing. | +| `mcp_oauth_grants` | [mcp_oauth_grants](../../packages/db/src/schema/public_mcp.ts); [public-mcp/oauth.ts](../../server/src/services/public-mcp/oauth.ts) | Any retained row, with no terminal exception. [public-mcp.test.ts](../../server/src/__tests__/public-mcp.test.ts) supplies domain fixtures. | Incoming-work: verify exact completion/authority/resource and dependent effects before narrowing. | +| `mcp_mutation_receipts` | [mcp_mutation_receipts](../../packages/db/src/schema/public_mcp.ts); [public-mcp/capabilities.ts](../../server/src/services/public-mcp/capabilities.ts) | Any retained row, with no terminal exception. [public-mcp.test.ts](../../server/src/__tests__/public-mcp.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `mcp_event_subscriptions` | [mcp_event_subscriptions](../../packages/db/src/schema/public_mcp.ts); [public-mcp/events.ts](../../server/src/services/public-mcp/events.ts) | Any retained row, with no terminal exception. [public-mcp.test.ts](../../server/src/__tests__/public-mcp.test.ts) supplies domain fixtures. | Incoming-work/deadline: verify exact completion/authority/resource and dependent effects before narrowing. | +| `mcp_event_deliveries` | [mcp_event_deliveries](../../packages/db/src/schema/public_mcp.ts); [public-mcp/events.ts](../../server/src/services/public-mcp/events.ts) | Any retained row, with no terminal exception. [public-mcp.test.ts](../../server/src/__tests__/public-mcp.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `mcp_attachment_uploads` | [mcp_attachment_uploads](../../packages/db/src/schema/public_mcp.ts); [public-mcp/file-transfers.ts](../../server/src/services/public-mcp/file-transfers.ts) | Any retained row, with no terminal exception. [public-mcp.test.ts](../../server/src/__tests__/public-mcp.test.ts) supplies domain fixtures. | History/resource/deadline: verify exact completion/authority/resource and dependent effects before narrowing. | +| `dot_agent_bindings` | [dot_agent_bindings](../../packages/db/src/schema/dot_runner.ts); [../modules/external-agents/dot.ts](../../server/src/modules/external-agents/dot.ts) | Any retained row, with no terminal exception. [dot-runner.test.ts](../../server/src/__tests__/dot-runner.test.ts) supplies domain fixtures. | Incoming-work: verify exact completion/authority/resource and dependent effects before narrowing. | +| `dot_runner_assignments` | [dot_runner_assignments](../../packages/db/src/schema/dot_runner.ts); [dot-runner-broker.ts](../../server/src/services/dot-runner-broker.ts) | Any retained row, with no terminal exception. [dot-runner.test.ts](../../server/src/__tests__/dot-runner.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `dot_runner_operations` | [dot_runner_operations](../../packages/db/src/schema/dot_runner.ts); [../modules/external-agents/native-broker.ts](../../server/src/modules/external-agents/native-broker.ts) | Any retained row, with no terminal exception. [dot-runner.test.ts](../../server/src/__tests__/dot-runner.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `dot_mailbox_items` | [dot_mailbox_items](../../packages/db/src/schema/dot_runner.ts); [public-mcp/events.ts](../../server/src/services/public-mcp/events.ts) | Any retained row, with no terminal exception. [dot-runner.test.ts](../../server/src/__tests__/dot-runner.test.ts) supplies domain fixtures. | Completed-history/ingress: verify exact completion/authority/resource and dependent effects before narrowing. | +| `muse_agent_bindings` | [muse_agent_bindings](../../packages/db/src/schema/muse_runner.ts); [../modules/external-agents/muse.ts](../../server/src/modules/external-agents/muse.ts) | Any retained row, with no terminal exception. [muse-control.test.ts](../../server/src/__tests__/muse-control.test.ts) supplies domain fixtures. | Incoming-work: verify exact completion/authority/resource and dependent effects before narrowing. | +| `muse_credentials` | [muse_credentials](../../packages/db/src/schema/muse_runner.ts); [muse-oauth.ts](../../server/src/services/muse-oauth.ts) | Any retained row, with no terminal exception. [muse-control.test.ts](../../server/src/__tests__/muse-control.test.ts) supplies domain fixtures. | Incoming-work: verify exact completion/authority/resource and dependent effects before narrowing. | +| `muse_runner_assignments` | [muse_runner_assignments](../../packages/db/src/schema/muse_runner.ts); [muse-runner-broker.ts](../../server/src/services/muse-runner-broker.ts) | Any retained row, with no terminal exception. [muse-terminal-settlement.test.ts](../../server/src/__tests__/muse-terminal-settlement.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `muse_runner_operations` | [muse_runner_operations](../../packages/db/src/schema/muse_runner.ts); [muse-runner-broker.ts](../../server/src/services/muse-runner-broker.ts) | Any retained row, with no terminal exception. [muse-terminal-settlement.test.ts](../../server/src/__tests__/muse-terminal-settlement.test.ts) supplies domain fixtures. | Completed-history: verify exact completion/authority/resource and dependent effects before narrowing. | +| `muse_mailbox_items` | [muse_mailbox_items](../../packages/db/src/schema/muse_runner.ts); [muse-assignment-follow-up.ts](../../server/src/services/muse-assignment-follow-up.ts) | Any retained row, with no terminal exception. [muse-terminal-settlement.test.ts](../../server/src/__tests__/muse-terminal-settlement.test.ts) supplies domain fixtures. | Completed-history/ingress: verify exact completion/authority/resource and dependent effects before narrowing. | +| `muse_input_deliveries` | [muse_input_deliveries](../../packages/db/src/schema/muse_runner.ts); [muse-runner-broker.ts](../../server/src/services/muse-runner-broker.ts) | Any retained row, with no terminal exception. [muse-control.test.ts](../../server/src/__tests__/muse-control.test.ts) supplies domain fixtures. | Completed-history/ingress: verify exact completion/authority/resource and dependent effects before narrowing. | +| `external_agent_holds` | [external_agent_holds](../../packages/db/src/schema/muse_runner.ts); [../modules/external-agents/native-broker.ts](../../server/src/modules/external-agents/native-broker.ts) | Any retained row, with no terminal exception. [muse-control.test.ts](../../server/src/__tests__/muse-control.test.ts) supplies domain fixtures. | Active external authority: verify exact completion/authority/resource and dependent effects before narrowing. | + +## Schedules and local reasons + +`routines.schedule` and `issues.schedule` reuse the existing [routine reader](../../server/src/services/idle-scheduled-work.ts) and [issue reader](../../server/src/services/idle-issue-schedule.ts) in the same snapshot when Cloud negotiates the respective capability. Due, malformed, unsupported, foreign-company, overflow, claimed or inconsistent schedules block. Definitions are never advanced/cleared during inspection. An earliest in-process timer is not an external alarm receipt. Routine and monitor proofs still require Cloud to persist the exact alarm and revalidate the owned generation before stop. Agent timers, watchdogs, plugin jobs, merge-confirmation checks, refresh duties, and other unqualified deadlines retain blockers. + +The issue reader can accept backlog, unassigned todo, explicit blocked/human-owned waits, untouched onboarding and exact response-driven waits. Live execution, conversations, ready dependency repair and unsupported review/recovery remain conservative. Healthy idle conversation semantics in product documentation do not themselves relax this reader's current `conversation_agent_id` fence; domain owners must reconcile that gap with proof. + +[Local inspection](../../server/src/services/idle-local-work.ts) has these fixed reasons: + +| Reason | Owning truth, producer, trigger and negative fixture | Remedy / dependency | +| --- | --- | --- | +| `local.startup` | `markIdleStartupComplete` in server startup; unknown until reconstruction completes. | Finish startup/recovery; readiness alone is insufficient. | +| `local.ingress_tracking` | HTTP/upgrade tracking installation; unknown before admission is installed. | Install existing admission adapter before accepting routes. | +| `local.automatic_backups` | Automatic-backup configuration; present unless the report has the existing checkpoint hook. | Backups package qualifies checkpoint/startup backup plus restore; keep backups enabled. | +| `local.tracked_work` | `task-admission.ts` active promise/token count and generation; token starts before work and ends only at settlement. | Let accepted request/scheduler/cleanup/write work settle; unknown writes retain holds. | +| `local.registered_probes` | Aggregate of six production registration owners listed below; present/unknown until each owner's obligation settles. | Owner recovery/resource proof; no generic timer exemption. | +| `local.accounting_spool` | `usage-receipts.ts`; any entry, including malformed/temp/write probe, blocks. | Durable accounting handoff and cleanup; empty/absent valid directory is clear. | +| `local.orphan_spool` | `sandbox-orphan-cleanup-spool.ts`; buffered or filesystem cleanup debt remains work. | Durable cleanup handoff and verified empty spool. | +| `local.registered_spools` | Additional configured spool directories registered by the same owners. | Every directory must be empty/absent; a malformed path/read failure is unknown. | +| `local.quota_recovery` | Company-grouped encrypted credential recovery journals; empty valid company directories are passive. Files, nested/invalid entries or bad roots block/unknown. | Existing quota recovery owns decryption/handoff; diagnostics never read file content or follow symlinks. | + +Registered-probe owners are `retained-ai-credential-cleanup.ts` (retained retire/cleanup), `local-ai-login.ts` (browser/deferred retirements), `native-runtime/native-session-executor.ts` (warm/closing sessions and checkpoint saves), `run-evidence-migration-work.ts` (unknown/incomplete/blocked S3 conversion), `provider-trace-retention-work.ts` (unknown discovery or **any future retention deadline**), and `index.ts` terminal workspace recovery (incomplete/failed startup pages). Future provider-trace retention is currently a local blocker; routine/monitor alarms do not cover it. Optional skill-package migration/retention is a distinct path which can wait for next ingress; do not conflate the two. + +`plugins.runtime_drain` reports the owning worker manager's existing aggregate proof. Startup locks, worker replacement/crash, missing SDK capability, live RPC/host/PTY/duplex routes, bad response or changed hold remain present/unknown. The collector never exposes the exempted worker IDs. Unsafe inspection is `not_checked`; it is never forced merely to complete diagnostics. + +`backup.checkpoint` is an action result, not read coverage. Diagnostic calls leave it `not_checked` and do not create dumps. Ordinary safety reads still run the existing fresh tracked dump/archive-verification/fsync protocol only after all other checks pass, require unchanged generation and retain the restart backup marker. Failed/slow/untracked/concurrent dump work does not authorize sleep. Storage persistence and real restoration belong to backup qualification. + +`inspection.hold`, `inspection.generation`, `durable.snapshot` and `local.inspection` describe observation integrity. They cannot convert unresolved work into a clear result. A malformed query result or inaccessible scope produces unknown, never an empty reason list presented as complete. + +## Non-DB traffic and ingress coverage + +These are separate qualification inputs. The local reason report does not pretend to observe a provider, Cloud router, another process or a browser from its database snapshot. + +| Traffic owner | Current source behavior | Trigger / quiet condition and required evidence | +| --- | --- | --- | +| [Announcements](../../server/src/services/announcement-feed.ts), [UI hook](../../ui/src/hooks/useAnnouncement.ts) | Feed refresh on demand with cache/failure cooldown; UI visibility/online events, no periodic server feed scan. | Accepted request uses normal HTTP tracking. Hidden tab suspends hook fetch; visible return requests content. Confirm actual browser/probe traffic during acceptance. | +| [Telemetry](../../server/src/telemetry.ts), [client](../../packages/shared/src/telemetry/client.ts) | First-party opt-out event path; 60-second flush timer, empty queue makes no send, queued events/retries can send. | Measure actual outbound sends and pending retry effects; an unref timer alone is not evidence of traffic or quiet. No telemetry policy change here. | +| [Observability](../../server/src/instrumentation.ts) | OpenTelemetry SDK/exporters only behind operator OTLP endpoint; absent endpoint is no-op. | With endpoint configured, measure exporter/buffer effects. Do not confuse this with first-party Telemetry or local run-log rows. | +| [Live-event WebSocket](../../server/src/realtime/live-events-ws.ts) | 30-second pings on existing clients; no clients means no ping traffic. Authentication work is tracked to settlement. | Measure provider socket policy and actual clients; active use may legitimately stay awake. | +| [Browser live updates](../../ui/src/context/LiveUpdatesProvider.tsx) | Hidden tab does not connect/poll in the visible-session effect. Visible disconnected clients retry WS and invalidate active queries on fallback interval; visible return repairs missed events. | Verify closed browser, hidden tab, visible idle and active use independently; page-specific query policies can add traffic. | +| [DB client](../../packages/db/src/client.ts), [pool settings](../DATABASE.md#client-tuning-optional) | Shared pool releases idle connections and reconnects on work. Open transaction/connection churn and actual worker SQL matter independently. | Observe connections without issuing tenant SQL during suspension window. Do not infer DB suspension from safety eligibility. | +| [Health](../../server/src/routes/health.ts), hosting probes | Health/control traffic remains admitted but tracked; external probe frequency is deployment configuration. | Record intentional observers; use provider/control metadata to observe sleep without waking it. | +| Direct APIs, webhooks, MCP, chat, Dot/Muse, preview/socket/provider transports | Admission and domain authorization are still required. Saved credentials/listeners stay conservative until exact stack-bound wake/admission/replay is supported. | Ingress owner proves during-drain/stop/cold-start delivery, revocation/expiry/wrong-tenant rejection and durable receipt/replay. Local quiet alone cannot prove it. | + +## Verification and handoff + +[Diagnostic unit tests](../../server/src/__tests__/idle-sleep-diagnostics.test.ts) compile the real 72-column query and cover simultaneous reasons, unchanged ordinary short circuits/shape, missing/stale owners, generation/expiry changes, unsafe plugin skip, backup nonexecution, malformed shape/types, thrown boundary failures and redaction. [Local tests](../../server/src/__tests__/idle-local-work.test.ts) inspect actual temporary filesystem states, simultaneous spools/probe failure, and no content reads. [Route tests](../../server/src/__tests__/instance-settings-routes.test.ts) preserve administrator/company/agent/Cloud/backup boundaries and opt-in behavior. + +[Real PostgreSQL fixtures](../../server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts) use the full migrated schema and actual predicates: settled history, simultaneous credential/debt/run/routine plus completed chat/transfer history, processed versus pending connector application, negotiated routine/monitor coverage, due schedule with unrelated retained rows, and malformed numeric config. They retain row identities and due times. [Existing safety regressions](../../server/src/__tests__/idle-sleep-safety.test.ts) continue to exercise consumed retries, coalesced wakes, cleanup/login proof, question handoffs, credentials, plugins, cross-company links and schedule generations. + +For a coordinator-provisioned disposable database, set `PAPERCLIP_IDLE_SLEEP_DIAGNOSTICS_TEST_DATABASE_URL` and `PAPERCLIP_IDLE_SLEEP_SAFETY_TEST_DATABASE_URL` to the explicit dedicated `*_test` URL and the existing `PAPERCLIP_TEST_POSTGRES_ADMIN_URL` safeguard. Run suites sequentially with one worker and bounded pools; close clients before coordinator cleanup. Normal CI retains the existing embedded fixture fallback. No diagnostic suite provisions or drops a DB when its explicit URL is set. + +The diagnostic does not mutate domain state, establish new external wake coverage, delete retained history, or claim a fleet rollout. History, incoming work, deadlines, backups, and assembled deployed acceptance remain separately owned follow-ups. Every proposed eligibility improvement must supply its exact domain completion and first-producer/reactivation evidence before the predicate changes. diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts index 8d14bb2f9da..87548d4b507 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts @@ -1986,6 +1986,7 @@ describe("ACPX runtime host", () => { it("retains a settled turn when shutdown cleanup must be retried", async () => { const fixture = await hostFixture(); + const credential = credentialLeaseFixture(fixture.root); let settleTurn!: (value: { stopReason: string }) => void; const turn = { ...runtimeTurn(), @@ -2000,6 +2001,8 @@ describe("ACPX runtime host", () => { if (failRuntimeClose) throw new Error("runtime cleanup failed"); }, }); + const dependencies = fixture.dependencies({ openRuntime: async () => runtime }); + dependencies.stageCredential = vi.fn(async () => credential); const host = await AcpxRuntimeHost.open( { ...fixture.options, @@ -2008,7 +2011,7 @@ describe("ACPX runtime host", () => { permissionMode: "approve-reads", environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, }, - fixture.dependencies({ openRuntime: async () => runtime }), + dependencies, ); host.startTurn({ text: "Complete the task.", requestId: "turn-1" }); @@ -2016,6 +2019,8 @@ describe("ACPX runtime host", () => { settleTurn({ stopReason: "end_turn" }); await expect(firstClose).rejects.toThrow(/cleanup failed/); expect(turn.cancel).toHaveBeenCalledOnce(); + expect(credential.close).not.toHaveBeenCalled(); + expect(host.isClosed()).toBe(false); failRuntimeClose = false; await expect( @@ -2023,6 +2028,8 @@ describe("ACPX runtime host", () => { ).resolves.toBeUndefined(); expect(turn.cancel).toHaveBeenCalledTimes(2); expect(runtime.close).toHaveBeenCalledTimes(2); + expect(credential.close).toHaveBeenCalledOnce(); + expect(host.isClosed()).toBe(true); }); it("closes a command lease that resolves after admission is aborted", async () => { @@ -2307,7 +2314,7 @@ describe("ACPX runtime host", () => { }, ACPX_LONG_WAIT_TEST_TIMEOUT_MS); }); -// Deadline tests exercise runtime admission and cleanup, not kernel lease +// Runtime lifecycle tests exercise admission and cleanup, not kernel lease // acquisition. Keep their credential owner deterministic: real lease staging // uses hashed loopback ports that unrelated parallel tests can occupy. The // credential and retained-owner integration tests still use real leases. diff --git a/server/src/__tests__/idle-local-work.test.ts b/server/src/__tests__/idle-local-work.test.ts index 4d8426796f8..c84844a281c 100644 --- a/server/src/__tests__/idle-local-work.test.ts +++ b/server/src/__tests__/idle-local-work.test.ts @@ -3,6 +3,8 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { inspectIdleSpool, inspectIdleQuotaRecovery, idleQuotaRecoveryPath, readIdleLocalWork, markIdleIngressTracked, markIdleStartupComplete, registerIdleSpoolDirectory, registerIdleLocalWorkProbe } from "../services/idle-local-work.js"; +import { idleAccountingSpoolPath } from "../services/idle-local-work.js"; +import { createIdleSleepDiagnostics } from "../services/idle-sleep-diagnostics.js"; let root: string; beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "idle-spool-")); }); @@ -66,6 +68,33 @@ describe("idle cleanup inspection", () => { expect(await readIdleLocalWork()).toBe("none"); }); + it("collects simultaneous local blockers and errors without reading entries", async () => { + vi.stubEnv("PAPERCLIP_HOME", root); + const orphan = path.join(root, "orphan-spool"); + vi.stubEnv("SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR", orphan); + markIdleIngressTracked(); markIdleStartupComplete({ scheduledBackups: true }); + const accounting = idleAccountingSpoolPath(); + await fs.mkdir(accounting, { recursive: true }); + await fs.mkdir(orphan); + await fs.writeFile(path.join(accounting, "private-receipt.json"), "malformed"); + await fs.writeFile(path.join(orphan, "partial.tmp"), "private resource"); + const unregister = registerIdleLocalWorkProbe(() => { throw new Error("private recovery failure"); }); + const diagnostic = createIdleSleepDiagnostics(); + const read = vi.spyOn(fs, "readFile"); + try { + expect(await readIdleLocalWork({ diagnostic: diagnostic.record })).toBe("unknown"); + expect(diagnostic.report().checks).toEqual(expect.arrayContaining([ + { reason: "local.automatic_backups", state: "present" }, + { reason: "local.registered_probes", state: "unknown" }, + { reason: "local.accounting_spool", state: "present" }, + { reason: "local.orphan_spool", state: "present" }, + ])); + expect(diagnostic.report().complete).toBe(false); + expect(JSON.stringify(diagnostic.report())).not.toMatch(/private|partial.tmp|orphan-spool/); + expect(read).not.toHaveBeenCalled(); + } finally { unregister(); markIdleStartupComplete({ scheduledBackups: false }); } + }); + }); describe("idle credential recovery inspection", () => { diff --git a/server/src/__tests__/idle-scheduled-work.test.ts b/server/src/__tests__/idle-scheduled-work.test.ts index 11b2c9d6b25..bc91dd7b86c 100644 --- a/server/src/__tests__/idle-scheduled-work.test.ts +++ b/server/src/__tests__/idle-scheduled-work.test.ts @@ -24,7 +24,7 @@ describe("bounded idle routine catalog", () => { const formatter = vi.spyOn(Intl, "DateTimeFormat"); try { expect(await readIdleRoutineSchedule(database(Array(10_001).fill(trigger)), "owner", now)) - .toEqual({ blocked: true }); + .toEqual({ blocked: true, inspectionComplete: false }); expect(formatter).not.toHaveBeenCalled(); } finally { formatter.mockRestore(); } }); diff --git a/server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts b/server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts new file mode 100644 index 00000000000..1f36be6bb81 --- /dev/null +++ b/server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts @@ -0,0 +1,130 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { agentApiKeys, agents, budgetPolicies, chatDeliveries, chatEndpoints, companies, + companyTransferRuns, connectionEventDeliveries, createDb, heartbeatRuns, issues, + routineTriggers, routines, toolApplications, toolConnections, type Db } from "@paperclipai/db"; +import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const now = Date.UTC(2026, 10, 2); +const ownerId = randomUUID(); +const hold = () => ({ ownerId, draining: true, startedAt: new Date(now - 1000), + expiresAt: new Date(now + 60_000), activeRuns: 0, pendingWakes: 0 }); +const localNone = async () => "none" as const; + +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_IDLE_SLEEP_DIAGNOSTICS_TEST_DATABASE_URL"); +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +if (!support.supported) console.warn(`Skipping idle sleep diagnostics Postgres tests: ${support.reason}`); +(support.supported ? describe : describe.skip)("existing-instance sleep blocker diagnostics", () => { + let db: Db; + let temporary: Awaited> | undefined; + beforeAll(async () => { + if (!externalDatabaseUrl) temporary = await startEmbeddedPostgresTestDatabase("idle-sleep-diagnostics-"); + db = createDb(externalDatabaseUrl ?? temporary!.connectionString, { maxConnections: 4 }); + }, 60_000); + afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + const read = (scheduled = false, monitors = false) => readIdleSleepSafety(db, hold, () => now, ownerId, + localNone, undefined, undefined, scheduled, monitors, { collectDiagnostics: true }); + const blockers = (report: Awaited>) => report.diagnostics?.checks + .filter(check => check.state === "present").map(check => check.reason); + + async function seed() { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Existing inventory fixture", issuePrefix: `DI${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "On-demand worker", role: "engineer", adapterType: "process", status: "idle" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", + status: "succeeded", finishedAt: new Date(now - 1000) }); + return { companyId, agentId, runId }; + } + + async function chatHistory(f: Awaited>) { + const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Slack fixture", type: "rest_api" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application!.id, + name: "Unconfigured Slack", uid: randomUUID(), connectionPurpose: "channel", transport: "chat_sdk", status: "draft", enabled: false }).returning(); + const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection!.id, + provider: "slack", publicId: randomUUID(), assignedAgentId: f.agentId, status: "draft", setup: { step: "provider_setup" } }).returning(); + const [delivery] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: endpoint!.id, + providerEventId: randomUUID(), deduplicationKey: randomUUID(), eventKind: "message", normalizedEvent: {}, + state: "processed", processedAt: new Date(now - 1000) }).returning(); + return delivery!; + } + + it("inspects every current predicate while retaining settled ordinary history", async () => { + const f = await seed(); + const report = await read(); + expect(report.backgroundWork).toBe("none"); + expect(report.diagnostics?.complete).toBe(true); + expect(blockers(report)).toEqual([]); + expect(report.diagnostics?.checks.filter(check => !check.reason.includes(".") || + ["agents.execution_or_timer", "agents.lifecycle", "agents.company_hold", "heartbeat_runs.budget_cancellation"].includes(check.reason))) + .toHaveLength(72); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId))).toHaveLength(1); + }); + + it("reports history, credentials, debt, running work and future configuration simultaneously", async () => { + const f = await seed(); + const history = await chatHistory(f); + await db.insert(agentApiKeys).values({ companyId: f.companyId, agentId: f.agentId, name: "Saved machine access", keyHash: randomUUID() }); + await db.insert(companyTransferRuns).values({ companyId: f.companyId, direction: "export", status: "completed", + actorKey: "fixture-board", containerRef: {}, idempotencyKey: randomUUID(), finishedAt: new Date(now - 2000) }); + await db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "company", scopeId: f.companyId, + windowKind: "calendar_month_utc", enforcementVersion: 2, enforcementDeliveredVersion: 1 }); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, invocationSource: "on_demand", status: "queued" }); + await db.insert(routines).values({ companyId: f.companyId, title: "Configured future duty", status: "active", assigneeAgentId: f.agentId }); + const report = await read(); + expect(report.backgroundWork).toBe("present"); + expect(report.diagnostics?.complete).toBe(true); + expect(blockers(report)).toEqual(["agent_api_keys", "budget_policies", "chat_deliveries", "company_transfer_runs", "heartbeat_runs", "routines"]); + expect(await db.select().from(chatDeliveries).where(eq(chatDeliveries.id, history.id))) + .toMatchObject([{ state: "processed", processedAt: history.processedAt }]); + expect(await db.select().from(agentApiKeys)).toHaveLength(1); + expect(await db.select().from(companyTransferRuns)).toMatchObject([{ status: "completed" }]); + }); + + it.each(["processed", "retry"] as const)("identifies retained %s chat rows without changing domain policy", async state => { + const delivery = await chatHistory(await seed()); + await db.update(chatDeliveries).set({ state }).where(eq(chatDeliveries.id, delivery.id)); + expect(blockers(await read())).toEqual(["chat_deliveries"]); + expect(await db.select().from(chatDeliveries)).toMatchObject([{ id: delivery.id, state }]); + }); + + it("distinguishes processed connector receipts from pending application without clearing history", async () => { + const f = await seed(); + const [receipt] = await db.insert(connectionEventDeliveries).values({ companyId: f.companyId, provider: "github", + providerDeliveryId: randomUUID(), event: "issues", status: "processed", processedAt: new Date(now - 1000) }).returning(); + expect(blockers(await read())).toEqual([]); + await db.update(connectionEventDeliveries).set({ status: "received", processedAt: null }).where(eq(connectionEventDeliveries.id, receipt!.id)); + expect(blockers(await read())).toEqual(["connection_event_deliveries"]); + expect(await db.select().from(connectionEventDeliveries)).toHaveLength(1); + }); + + it("collects schedule and unrelated history reasons from the same negotiated inspection", async () => { + const f = await seed(); + await chatHistory(f); + const [routine] = await db.insert(routines).values({ companyId: f.companyId, title: "Future routine", status: "active", assigneeAgentId: f.agentId }).returning(); + const [trigger] = await db.insert(routineTriggers).values({ companyId: f.companyId, routineId: routine!.id, kind: "schedule", + cronExpression: "*/15 * * * *", timezone: "UTC", nextRunAt: new Date(now + 600_000), enabled: true, archived: false, setupPending: false }).returning(); + await db.insert(issues).values({ companyId: f.companyId, title: "Passive backlog", status: "backlog" }); + expect(blockers(await read())).toEqual(["chat_deliveries", "issues", "routines"]); + expect(blockers(await read(true, true))).toEqual(["chat_deliveries"]); + await db.update(routineTriggers).set({ nextRunAt: new Date(now - 1000) }).where(eq(routineTriggers.id, trigger!.id)); + const due = await read(true, true); + expect(blockers(due)).toEqual(["chat_deliveries", "routines.schedule"]); + expect(await db.select().from(routineTriggers)).toMatchObject([{ nextRunAt: new Date(now - 1000) }]); + }); + + it("fails closed on a real malformed SQL configuration without leaking values", async () => { + const f = await seed(); + await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: true, intervalSec: "private-bad-interval" } } }) + .where(eq(agents.id, f.agentId)); + const report = await read(); + expect(report.backgroundWork).toBe("unknown"); + expect(report.diagnostics?.complete).toBe(false); + expect(report.diagnostics?.checks).toContainEqual({ reason: "durable.snapshot", state: "unknown" }); + expect(JSON.stringify(report)).not.toMatch(/private-bad-interval|numeric|SELECT|runtime_config/); + }); +}); diff --git a/server/src/__tests__/idle-sleep-diagnostics.test.ts b/server/src/__tests__/idle-sleep-diagnostics.test.ts new file mode 100644 index 00000000000..58c6cc3bdba --- /dev/null +++ b/server/src/__tests__/idle-sleep-diagnostics.test.ts @@ -0,0 +1,217 @@ +import { PgDialect } from "drizzle-orm/pg-core"; +import { readFileSync } from "node:fs"; +import type { SQL } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { describe, expect, it, vi } from "vitest"; +import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js"; +import { beginIdleTrackedWork } from "../services/task-admission.js"; + +const now = Date.UTC(2026, 10, 2); +const ownerId = "inventory-owned-hold"; +const hold = (): IdleSleepDrainStatus => ({ ownerId, draining: true, startedAt: new Date(now - 1000), + expiresAt: new Date(now + 60_000), activeRuns: 0, pendingWakes: 0 }); +const localNone = async () => "none" as const; +const collect = { collectDiagnostics: true } as const; + +function database(present: string[] = [], change?: (row: Record) => unknown, + schedules?: { routines?: unknown[]; issues?: unknown[] }) { + let keys: string[] = []; + const execute = vi.fn(async (statement: SQL) => { + const query = new PgDialect().sqlToQuery(statement).sql; + if (query.startsWith("SET LOCAL")) return []; + if (query.includes("FROM routine_triggers t JOIN routines r")) return schedules?.routines ?? []; + if (query.includes("FROM issues i JOIN companies c")) return schedules?.issues ?? []; + if (query.includes("WHERE status NOT IN ('active', 'paused', 'archived')")) return [{ blocked: false }]; + if (query.endsWith(" AS blocked")) return [{ blocked: present.length > 0 }]; + keys = [...query.matchAll(/ AS "([^"]+)"(?=, EXISTS |$)/g)].map(match => match[1]!); + const row = Object.fromEntries(keys.map(key => [key, present.includes(key)])); + return change ? change(row) : [row]; + }); + const transaction = vi.fn(async (run: (tx: unknown) => Promise) => run({ execute })); + return { db: { transaction } as unknown as Db, transaction, execute, keys: () => keys }; +} + +describe("owned idle sleep diagnostics", () => { + it("reports simultaneous blockers from every predicate in one safe snapshot", async () => { + const f = database(["agent_api_keys", "heartbeat_runs", "chat_deliveries", "muse_mailbox_items"]); + const result = await readIdleSleepSafety(f.db, hold, () => now, ownerId, localNone, + undefined, undefined, false, false, collect); + expect(result.backgroundWork).toBe("present"); + expect(result.diagnostics?.complete).toBe(true); + expect(result.diagnostics?.checks.filter(check => check.state === "present").map(check => check.reason)) + .toEqual(["agent_api_keys", "chat_deliveries", "heartbeat_runs", "muse_mailbox_items"]); + expect(f.keys()).toHaveLength(72); + expect(new Set(f.keys()).size).toBe(f.keys().length); + const inventory = readFileSync(new URL("../../../doc/plans/2026-11-02-sleep-blocker-inventory.md", import.meta.url), "utf8"); + const documented = [...inventory.matchAll(/^\| `([a-z0-9_.]+)` \|/gm)].map(match => match[1]); + expect(f.keys().filter(key => !documented.includes(key))).toEqual([]); + expect(f.transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" }); + expect(f.execute).toHaveBeenCalledTimes(2); + }); + + it("can report each registered predicate independently, including every retained-table family", async () => { + const active: string[] = []; + const f = database(active); + const read = () => readIdleSleepSafety(f.db, hold, () => now, ownerId, localNone, + undefined, undefined, false, false, collect); + await read(); + for (const key of f.keys()) { + active.splice(0, active.length, key); + const result = await read(); + expect(result.backgroundWork, key).toBe("present"); + expect(result.diagnostics?.checks.filter(check => check.state === "present"), key) + .toEqual([{ reason: key, state: "present" }]); + } + }); + + const routine = { id: "trigger", routine_id: "routine", company_id: "company", kind: "schedule", + setup_pending: false, cron_expression: "*/15 * * * *", timezone: "UTC", scoped: true, + next_run_at: new Date(now + 600_000), routine_updated: new Date(now), trigger_updated: new Date(now) }; + const issue = { id: "issue", company_id: "company", status: "backlog", updated_at: new Date(now), + assignee_agent_id: "agent", assignee_user_id: null, checkout_run_id: null, execution_run_id: null, + execution_policy: null, execution_state: null, unblock_descriptor: null, conversation_agent_id: null, + monitor_next_check_at: null, monitor_wake_requested_at: null, monitor_attempt_count: 0, + company_status: "active", company_updated: new Date(now), agent_status: "idle", agent_updated: new Date(now), + scoped: true, unsafe_dependencies: false, dependency_ready: false, untouched_onboarding: false, + response_driven_wait: false, unblock_request_finished: false, response_wait_generation: null }; + const scheduledRead = (f: ReturnType, diagnostics = true) => readIdleSleepSafety(f.db, hold, + () => now, ownerId, localNone, undefined, undefined, true, true, diagnostics ? collect : undefined); + + it.each(["routines", "issues"] as const)("marks an overflowing %s inspection unknown without relaxing ordinary eligibility", async family => { + const f = database([], undefined, { [family]: Array(10_001).fill(family === "routines" ? routine : issue) }); + expect(await scheduledRead(f, false)).toEqual({ version: 1, backgroundWork: "present" }); + const result = await scheduledRead(f); + expect(result.backgroundWork).toBe("unknown"); + expect(result.diagnostics?.complete).toBe(false); + expect(result.diagnostics?.checks).toContainEqual({ reason: `${family}.schedule`, state: "unknown" }); + }); + + it.each(["routines", "issues"] as const)("fully inspects a %s catalog at the 10,000-row bound", async family => { + const f = database([], undefined, { [family]: Array(10_000).fill(family === "routines" ? routine : issue) }); + const result = await scheduledRead(f); + expect(result.backgroundWork).toBe("none"); + expect(result.diagnostics?.complete).toBe(true); + expect(result.diagnostics?.checks).toContainEqual({ reason: `${family}.schedule`, state: "none" }); + }); + + it.each(["routines", "issues"] as const)("retains an actual %s blocker as known work", async family => { + const f = database([], undefined, { [family]: [family === "routines" ? { ...routine, next_run_at: new Date(now) } + : { ...issue, status: "todo" }] }); + const result = await scheduledRead(f); + expect(result.backgroundWork).toBe("present"); + expect(result.diagnostics?.complete).toBe(true); + expect(result.diagnostics?.checks).toContainEqual({ reason: `${family}.schedule`, state: "present" }); + }); + + it.each(["routines", "issues"] as const)("keeps known durable work present alongside an incomplete %s inspection", async family => { + const f = database(["heartbeat_runs"], undefined, { [family]: Array(10_001).fill(family === "routines" ? routine : issue) }); + const result = await scheduledRead(f); + expect(result.backgroundWork).toBe("present"); + expect(result.diagnostics?.complete).toBe(false); + expect(result.diagnostics?.checks).toEqual(expect.arrayContaining([ + { reason: "heartbeat_runs", state: "present" }, { reason: `${family}.schedule`, state: "unknown" }, + ])); + }); + + it("preserves ordinary local and plugin short circuits and response shape", async () => { + const f = database(); + const plugins = vi.fn(async () => ({ backgroundWork: "present" as const, pluginIds: [] })); + expect(await readIdleSleepSafety(f.db, hold, () => now, ownerId, async () => "present", plugins)) + .toEqual({ version: 1, backgroundWork: "present" }); + expect(plugins).not.toHaveBeenCalled(); + expect(f.transaction).not.toHaveBeenCalled(); + expect(await readIdleSleepSafety(f.db, hold, () => now, ownerId, localNone, plugins)) + .toEqual({ version: 1, backgroundWork: "present" }); + expect(f.transaction).not.toHaveBeenCalled(); + expect(await readIdleSleepSafety(f.db, hold, () => now, ownerId, localNone)) + .toEqual({ version: 1, backgroundWork: "none" }); + }); + + it("continues safe inspection after local and plugin blockers without starting a backup", async () => { + const f = database(["company_secret_proposals"]); + const backup = vi.fn(async () => true); + const plugins = vi.fn(async () => ({ backgroundWork: "present" as const, pluginIds: [] })); + const result = await readIdleSleepSafety(f.db, hold, () => now, ownerId, async record => { + record?.("local.accounting_spool", "present"); + record?.("local.orphan_spool", "present"); + return "present"; + }, plugins, backup, false, false, collect); + expect(result.backgroundWork).toBe("present"); + expect(result.diagnostics?.checks).toEqual(expect.arrayContaining([ + { reason: "company_secret_proposals", state: "present" }, + { reason: "local.accounting_spool", state: "present" }, + { reason: "local.orphan_spool", state: "present" }, + { reason: "plugins.runtime_drain", state: "present" }, + { reason: "backup.checkpoint", state: "not_checked" }, + ])); + expect(backup).not.toHaveBeenCalled(); + }); + + it("does not run checkpoint actions or treat unperformed actions as quiet", async () => { + const f = database(); + const backup = vi.fn(async () => true); + const result = await readIdleSleepSafety(f.db, hold, () => now, ownerId, localNone, + undefined, backup, false, false, collect); + expect(result.backgroundWork).toBe("unknown"); + expect(result.diagnostics?.complete).toBe(true); + expect(result.diagnostics?.checks).toContainEqual({ reason: "backup.checkpoint", state: "not_checked" }); + expect(backup).not.toHaveBeenCalled(); + }); + + it.each([undefined, "wrong-owner"])("never opens the DB for diagnostic owner %s", async owner => { + const f = database(); + const result = await readIdleSleepSafety(f.db, hold, () => now, owner, localNone, + undefined, undefined, false, false, collect); + expect(result.backgroundWork).toBe("unknown"); + expect(result.diagnostics?.complete).toBe(false); + expect(f.transaction).not.toHaveBeenCalled(); + }); + + it("skips unsafe plugin RPCs while accepted local work is active", async () => { + const f = database(); + const plugins = vi.fn(); + const finish = beginIdleTrackedWork(); + try { + const result = await readIdleSleepSafety(f.db, hold, () => now, ownerId, async () => "present", + plugins, undefined, false, false, collect); + expect(result.backgroundWork).toBe("unknown"); + expect(result.diagnostics?.complete).toBe(false); + expect(result.diagnostics?.checks).toContainEqual({ reason: "plugins.runtime_drain", state: "not_checked" }); + expect(plugins).not.toHaveBeenCalled(); + } finally { finish(); } + }); + + it.each(["owner", "expiry", "generation"])("invalidates the observation after changed %s", async changed => { + const f = database(); + let current = hold(); + let clock = now; + let reads = 0; + const result = await readIdleSleepSafety(f.db, () => current, () => clock, ownerId, async () => { + if (++reads === 2) { + if (changed === "owner") current = { ...current, ownerId: "replacement" }; + if (changed === "expiry") clock += 60_001; + if (changed === "generation") { const finish = beginIdleTrackedWork(); finish(); } + } + return "none"; + }, undefined, undefined, false, false, collect); + expect(result.backgroundWork).toBe("unknown"); + expect(result.diagnostics?.complete).toBe(false); + }); + + it.each([ + (row: Record) => [{ ...row, heartbeat_runs: "false" }], + (row: Record) => [{ ...row, extra_private_field: "sensitive" }], + (row: Record) => { delete row.heartbeat_runs; return [row]; }, + () => [], + (row: Record) => [row, row], + () => { throw new Error("private SQL, credential, spool path"); }, + ])("fails closed and redacts malformed/unavailable durable reads", async change => { + const f = database([], change); + const result = await readIdleSleepSafety(f.db, hold, () => now, ownerId, localNone, + undefined, undefined, false, false, collect); + expect(result.backgroundWork).toBe("unknown"); + expect(result.diagnostics?.complete).toBe(false); + expect(result.diagnostics?.checks).toContainEqual({ reason: "durable.snapshot", state: "unknown" }); + expect(JSON.stringify(result)).not.toMatch(/sensitive|private SQL|credential,|extra_private/); + }); +}); diff --git a/server/src/__tests__/idle-sleep-safety.test.ts b/server/src/__tests__/idle-sleep-safety.test.ts index 40a93b09794..b14a814fdac 100644 --- a/server/src/__tests__/idle-sleep-safety.test.ts +++ b/server/src/__tests__/idle-sleep-safety.test.ts @@ -10,6 +10,7 @@ import { projects, projectWorkspaces, routines, routineTriggers, toolApplications, toolConnections, workspaceOperations, executionWorkspaceRuntimeLeases, type Db, } from "@paperclipai/db"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js"; const now = Date.UTC(2020, 0, 1); @@ -162,18 +163,19 @@ describe("idle sleep admission and local work", () => { }); }); -const support = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_IDLE_SLEEP_SAFETY_TEST_DATABASE_URL"); +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${support.reason}`); (support.supported ? describe : describe.skip)("idle sleep durable work", () => { let db: ReturnType; let database: Awaited>; beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("idle-sleep-safety-"); - db = createDb(database.connectionString); + if (!externalDatabaseUrl) database = await startEmbeddedPostgresTestDatabase("idle-sleep-safety-"); + db = createDb(externalDatabaseUrl ?? database.connectionString, { maxConnections: 4 }); }, 60_000); afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); }); - afterAll(async () => { await database?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal); const readScheduled = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true); diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.ts index 7c63fe704b7..08123e23a55 100644 --- a/server/src/__tests__/instance-settings-routes.test.ts +++ b/server/src/__tests__/instance-settings-routes.test.ts @@ -1132,6 +1132,28 @@ describe("instance settings routes", () => { expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus); }); + it("collects fixed-key diagnostics only through the privileged safety report", async () => { + mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus); + mockReadIdleSleepSafety.mockResolvedValue({ version: 1, backgroundWork: "present", + diagnostics: { version: 1, complete: true, checks: [{ reason: "agent_api_keys", state: "present" }] } }); + await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepDiagnostics=1").expect(200); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + const response = await request(createApp(adminActor)) + .get("/api/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&ownerId=fixture-owner").expect(200); + expect(response.body.idleSleepSafety.diagnostics.checks).toEqual([{ reason: "agent_api_keys", state: "present" }]); + expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, + "fixture-owner", expect.any(Function), undefined, undefined, false, false, { collectDiagnostics: true }); + mockReadIdleSleepSafety.mockClear(); + for (const actor of [nonAdminActor, { type: "agent", agentId: "agent-1", companyId: "company-1", source: "api_key" }]) { + await request(createApp(actor)) + .get("/api/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&ownerId=fixture-owner").expect(403); + } + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + await request(createApp(adminActor)) + .get("/api/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&scheduledWake=2&ownerId=fixture-owner").expect(403); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + }); + it.each(["1", "2"])("reserves scheduling capability %s for verified Cloud control callers", async version => { mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus); for (const source of ["session", "cloud_tenant", "local_implicit"]) { diff --git a/server/src/__tests__/issue-recovery-assignments.test.ts b/server/src/__tests__/issue-recovery-assignments.test.ts index 46ee33936ac..e91b0b5adfe 100644 --- a/server/src/__tests__/issue-recovery-assignments.test.ts +++ b/server/src/__tests__/issue-recovery-assignments.test.ts @@ -14,7 +14,9 @@ describeIssueRecovery("issue recovery production producers and assignments", () await finish(issue.id); await quiet(); }, 90_000); it("fences nested assignment writes until outer commit and ignores rollback", async () => { - const f = await fixture(); await start(); await quiet(); + const f = await fixture(); + const providerFailures = await state.completingProvider(f); + await start(); await quiet(); let committedId = ""; await state.db.transaction(async outer => { await outer.transaction(async nested => { @@ -33,6 +35,8 @@ describeIssueRecovery("issue recovery production producers and assignments", () })).rejects.toThrow("rollback control"); await quiet(); expect(await state.db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0); + expect(providerFailures).toEqual([]); + expect(await state.db.select().from(heartbeatRuns)).toHaveLength(1); }, 90_000); it("retains the first committed assignment during drain and executes on resume", async () => { const f = await fixture(); await start(); await quiet(); state.draining = true; diff --git a/server/src/routes/instance-settings.ts b/server/src/routes/instance-settings.ts index b8072b8b308..66123a51e52 100644 --- a/server/src/routes/instance-settings.ts +++ b/server/src/routes/instance-settings.ts @@ -334,8 +334,9 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag } const idleSleepSafety = await readIdleSleepSafety(db, () => heartbeat.getTaskDrainStatus(), Date.now, typeof req.query.ownerId === "string" ? req.query.ownerId : undefined, - () => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup) }), - pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors); + diagnostic => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup), diagnostic }), + pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors, + ...(req.query.idleSleepDiagnostics === "1" ? [{ collectDiagnostics: true }] as const : [] as const)); res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety }); return; } diff --git a/server/src/services/idle-issue-schedule.ts b/server/src/services/idle-issue-schedule.ts index c234403e873..5037c099d97 100644 --- a/server/src/services/idle-issue-schedule.ts +++ b/server/src/services/idle-issue-schedule.ts @@ -15,6 +15,7 @@ const record = (value: unknown): Record | null => export async function readIdleIssueSchedule(tx: Pick, ownerId: string, now: number): Promise<{ blocked: boolean; + inspectionComplete?: false; scheduledWake?: ScheduledWake; generation?: string; }> { @@ -53,7 +54,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st FROM issues i JOIN companies c ON c.id = i.company_id LEFT JOIN agents a ON a.id = i.assignee_agent_id AND a.company_id = i.company_id WHERE i.status NOT IN ('done', 'cancelled') ORDER BY i.id LIMIT ${limit + 1}`); - if (rows.length > limit) return { blocked: true }; + if (rows.length > limit) return { blocked: true, inspectionComplete: false }; let earliest = Infinity; for (const row of rows) { const policy = record(row.execution_policy); diff --git a/server/src/services/idle-local-work.ts b/server/src/services/idle-local-work.ts index 95b6549ebbc..6034d6515d8 100644 --- a/server/src/services/idle-local-work.ts +++ b/server/src/services/idle-local-work.ts @@ -2,6 +2,7 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { idleWorkSnapshot } from "./task-admission.js"; +import type { IdleSleepDiagnosticSink } from "./idle-sleep-diagnostics.js"; export type IdleLocalWork = "none" | "present" | "unknown"; const spoolDirectories = new Set(); @@ -61,21 +62,50 @@ export async function inspectIdleQuotaRecovery(directory: string): Promise { - if (!startupComplete || !ingressTracked) return "unknown"; +export async function readIdleLocalWork(options: { backupCheckpoint?: boolean; diagnostic?: IdleSleepDiagnosticSink } = {}): Promise { + const record = options.diagnostic; + const states: IdleLocalWork[] = []; + record?.("local.startup", startupComplete ? "none" : "unknown"); + record?.("local.ingress_tracking", ingressTracked ? "none" : "unknown"); + if (!startupComplete || !ingressTracked) { + if (!record) return "unknown"; + states.push("unknown"); + } // This exemption belongs only to a scan which will finish a new backup // under the same admission hold before it can return an empty report. - if ((scheduledBackups && !options.backupCheckpoint) || idleWorkSnapshot().active !== 0) return "present"; + const backup = scheduledBackups && !options.backupCheckpoint; + const active = idleWorkSnapshot().active !== 0; + record?.("local.automatic_backups", backup ? "present" : "none"); + record?.("local.tracked_work", active ? "present" : "none"); + if (backup || active) { + if (!record) return "present"; + states.push("present"); + } + let probes: IdleLocalWork; try { const local = [...localWorkProbes].map(probe => probe()); - if (local.includes("unknown")) return "unknown"; - if (local.includes("present")) return "present"; - } catch { return "unknown"; } - const directories = new Set([...spoolDirectories, idleAccountingSpoolPath(), idleOrphanSpoolPath()]); + probes = local.some(state => state !== "none" && state !== "present") ? "unknown" + : local.includes("present") ? "present" : "none"; + } catch { probes = "unknown"; } + record?.("local.registered_probes", probes); + if (probes !== "none" && !record) return probes; + states.push(probes); + const accounting = idleAccountingSpoolPath(); + const orphan = idleOrphanSpoolPath(); + const directories = new Set([...spoolDirectories, accounting, orphan]); const results = await Promise.all([ - ...[...directories].map(inspectIdleSpool), - inspectIdleQuotaRecovery(idleQuotaRecoveryPath()), + ...[...directories].map(async directory => { + const state = await inspectIdleSpool(directory); + record?.(directory === accounting ? "local.accounting_spool" + : directory === orphan ? "local.orphan_spool" : "local.registered_spools", state); + return state; + }), + inspectIdleQuotaRecovery(idleQuotaRecoveryPath()).then(state => { + record?.("local.quota_recovery", state); + return state; + }), ]); - if (results.includes("unknown")) return "unknown"; - return results.includes("present") ? "present" : "none"; + states.push(...results); + if (states.includes("unknown")) return "unknown"; + return states.includes("present") ? "present" : "none"; } diff --git a/server/src/services/idle-scheduled-work.ts b/server/src/services/idle-scheduled-work.ts index 6e9ed7f670b..f7f8c2b4b1d 100644 --- a/server/src/services/idle-scheduled-work.ts +++ b/server/src/services/idle-scheduled-work.ts @@ -15,6 +15,7 @@ export interface ScheduledWake { export async function readIdleRoutineSchedule(tx: Pick, ownerId: string, now: number): Promise<{ blocked: boolean; + inspectionComplete?: false; scheduledWake?: ScheduledWake; }> { const unknown = await tx.execute<{ blocked: boolean }>(sql`SELECT EXISTS ( @@ -33,7 +34,7 @@ export async function readIdleRoutineSchedule(tx: Pick, ownerId: FROM routine_triggers t JOIN routines r ON r.id = t.routine_id WHERE r.status = 'active' AND t.enabled AND NOT t.archived ORDER BY t.id LIMIT ${limit + 1}`); - if (rows.length > limit) return { blocked: true }; + if (rows.length > limit) return { blocked: true, inspectionComplete: false }; let earliest = Infinity; const validTimezones = new Set(); const validCrons = new Set(); diff --git a/server/src/services/idle-sleep-diagnostics.ts b/server/src/services/idle-sleep-diagnostics.ts new file mode 100644 index 00000000000..cab05990adf --- /dev/null +++ b/server/src/services/idle-sleep-diagnostics.ts @@ -0,0 +1,46 @@ +export type IdleSleepCheckState = "none" | "present" | "unknown" | "not_checked"; +export type IdleSleepDiagnosticSink = (reason: string, state: IdleSleepCheckState) => void; + +export interface IdleSleepDiagnostics { + version: 1; + complete: boolean; + checks: { reason: string; state: IdleSleepCheckState }[]; +} + +export const IDLE_LOCAL_DIAGNOSTIC_REASONS = [ + "local.startup", "local.ingress_tracking", "local.automatic_backups", "local.tracked_work", + "local.registered_probes", "local.accounting_spool", "local.orphan_spool", + "local.registered_spools", "local.quota_recovery", +] as const; + +export function createIdleSleepDiagnostics() { + const states = new Map(); + const allowed = new Set([ + ...IDLE_LOCAL_DIAGNOSTIC_REASONS, + "inspection.hold", "inspection.generation", "durable.snapshot", "local.inspection", + "plugins.runtime_drain", "routines.schedule", "issues.schedule", "backup.checkpoint", + ]); + const rank = { none: 0, not_checked: 1, present: 2, unknown: 3 } as const; + const record: IdleSleepDiagnosticSink = (reason, state) => { + if (!allowed.has(reason) || !Object.hasOwn(rank, state)) throw new Error("Invalid idle diagnostic check"); + const previous = states.get(reason); + if (previous === undefined || rank[state] > rank[previous]) states.set(reason, state); + }; + return { + record, + expect(reasons: string[]) { + for (const reason of reasons) { + if (allowed.has(reason)) throw new Error("Duplicate idle diagnostic reason"); + allowed.add(reason); + } + }, + report(): IdleSleepDiagnostics { + return { + version: 1, + complete: [...states].every(([reason, state]) => reason === "backup.checkpoint" + || (state !== "unknown" && state !== "not_checked")), + checks: [...states].sort(([a], [b]) => a.localeCompare(b)).map(([reason, state]) => ({ reason, state })), + }; + }, + }; +} diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index c0d58a64bdd..c773958a546 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -4,7 +4,8 @@ import { readIdleRoutineSchedule, type ScheduledWake } from "./idle-scheduled-wo import { combineIdleSchedules, readIdleIssueSchedule } from "./idle-issue-schedule.js"; import { connectionDeliveryWaitingForResponse, finishedIdleRun as finishedRun, finishedIdleWake, finishedFailedIdleLease as finishedFailedLease, passiveBoardRecovery } from "./idle-work-proof.js"; import type { Db } from "@paperclipai/db"; -import { sql } from "drizzle-orm"; +import { sql, type SQLWrapper } from "drizzle-orm"; +import { createIdleSleepDiagnostics, type IdleSleepDiagnosticSink, type IdleSleepDiagnostics } from "./idle-sleep-diagnostics.js"; import { verifiedResourceCleanup } from "./cleanup-verification-proof.js"; import { companyHoldMismatch } from "../modules/agent-lifecycle/index.js"; import { pendingBudgetRunCancellationPredicate } from "./heartbeat/budget-cancellation.js"; @@ -23,6 +24,7 @@ export interface IdleSleepSafety { version: 1 | 2; backgroundWork: "none" | "present" | "unknown"; scheduledWake?: ScheduledWake; + diagnostics?: IdleSleepDiagnostics; } export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) => Promise<{ @@ -40,50 +42,52 @@ export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) // Other background features remain awake until their work and inbound events // have a durable wake path. In particular, an empty in-memory run set is not // evidence that a restarted process has no persisted work to recover. +const workCheck = (reason: string, query: string | SQLWrapper) => ({ reason, query: typeof query === "string" ? sql.raw(query) : query }); + const WORK_CHECKS = [ - `SELECT 1 FROM agents WHERE status = 'running' OR + workCheck("agents.execution_or_timer", `SELECT 1 FROM agents WHERE status = 'running' OR (status NOT IN ('terminated', 'paused', 'pending_approval') AND runtime_config->'heartbeat'->>'enabled' = 'true' AND - COALESCE((runtime_config->'heartbeat'->>'intervalSec')::numeric, 0) > 0)`, - `SELECT 1 FROM heartbeat_runs WHERE + COALESCE((runtime_config->'heartbeat'->>'intervalSec')::numeric, 0) > 0)`), + workCheck("heartbeat_runs", `SELECT 1 FROM heartbeat_runs WHERE status NOT IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted') OR (scheduled_retry_at IS NOT NULL AND (finished_at IS NULL OR scheduled_retry_at > finished_at)) - OR cost_accounting_pending`, - `SELECT 1 FROM agent_instruction_working_copies WHERE + OR cost_accounting_pending`), + workCheck("agent_instruction_working_copies", `SELECT 1 FROM agent_instruction_working_copies WHERE state NOT IN ('saved', 'unchanged', 'resolved', 'unavailable', 'conflict') OR process_stopped_at IS NULL OR next_attempt_at IS NOT NULL - OR receipt ? 'baseline' OR receipt->>'cleanupPending' = 'true'`, - `SELECT 1 FROM sandbox_creation_operations WHERE state = 'pending'`, - `SELECT 1 FROM budget_reservations WHERE state NOT IN ('settled', 'released')`, - `SELECT 1 FROM budget_policies WHERE enforcement_version > enforcement_delivered_version`, - `SELECT 1 FROM agents WHERE lifecycle_state IN - ('preparing', 'verifying', 'pausing', 'resuming', 'terminating', 'cleaning_up')`, - `SELECT 1 FROM decision_invocations WHERE status = 'running'`, - `SELECT 1 FROM prompt_injection_invocations WHERE status = 'running'`, + OR receipt ? 'baseline' OR receipt->>'cleanupPending' = 'true'`), + workCheck("sandbox_creation_operations", `SELECT 1 FROM sandbox_creation_operations WHERE state = 'pending'`), + workCheck("budget_reservations", `SELECT 1 FROM budget_reservations WHERE state NOT IN ('settled', 'released')`), + workCheck("budget_policies", `SELECT 1 FROM budget_policies WHERE enforcement_version > enforcement_delivered_version`), + workCheck("agents.lifecycle", `SELECT 1 FROM agents WHERE lifecycle_state IN + ('preparing', 'verifying', 'pausing', 'resuming', 'terminating', 'cleaning_up')`), + workCheck("decision_invocations", `SELECT 1 FROM decision_invocations WHERE status = 'running'`), + workCheck("prompt_injection_invocations", `SELECT 1 FROM prompt_injection_invocations WHERE status = 'running'`), // Persistent machine credentials can receive work without a human opening // the instance. They need an ingress wake contract before we can sleep. - `SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`, - `SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL - AND (expires_at IS NULL OR expires_at > now())`, + workCheck("agent_api_keys", `SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`), + workCheck("board_api_keys", `SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL + AND (expires_at IS NULL OR expires_at > now())`), // Coalescing consumes a request into another run. Retain it as work until // both the request and that exact company/agent's run have finished. - `SELECT 1 FROM agent_wakeup_requests w WHERE + workCheck("agent_wakeup_requests", `SELECT 1 FROM agent_wakeup_requests w WHERE w.status NOT IN ('completed', 'failed', 'cancelled', 'skipped', 'timed_out', 'coalesced') OR (w.status = 'coalesced' AND (w.finished_at IS NULL OR NOT EXISTS ( SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_id AND r.company_id = w.company_id AND r.agent_id = w.agent_id - AND ${finishedRun("r")})))`, - `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`, + AND ${finishedRun("r")})))`), + workCheck("issues", `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`), // A completed issue can still need its first watchdog review, including a // retry after immediate evaluation failed before creating a review/run. - `SELECT 1 FROM issue_watchdogs WHERE status <> 'disabled'`, - `SELECT 1 FROM workspace_operations WHERE status NOT IN ('succeeded', 'failed', 'cancelled')`, + workCheck("issue_watchdogs", `SELECT 1 FROM issue_watchdogs WHERE status <> 'disabled'`), + workCheck("workspace_operations", `SELECT 1 FROM workspace_operations WHERE status NOT IN ('succeeded', 'failed', 'cancelled')`), // A local directory has no independent worker. Keep unfamiliar providers, // runtime/command configuration, reopen fences and due cleanup awake. Runs, // operations, runtime services, retained leases and backup work are checked // separately; inspecting the row never deletes or shortens retained data. - `SELECT 1 FROM execution_workspaces w WHERE ( + workCheck("execution_workspaces", `SELECT 1 FROM execution_workspaces w WHERE ( w.provider_type IN ('local_fs', 'git_worktree') AND w.strategy_type IN ('project_primary', 'git_worktree') AND w.mode IN ('inherit', 'shared_workspace', 'isolated_workspace', 'operator_branch', 'reuse_existing', 'agent_default') @@ -107,16 +111,16 @@ const WORK_CHECKS = [ AND COALESCE(w.metadata->'config'->>'runtimeProvisionCommand', '') = '' AND COALESCE(w.metadata->'config'->>'teardownCommand', '') = '' AND COALESCE(w.metadata->'config'->>'cleanupCommand', '') = '')) - ) IS NOT TRUE`, - `SELECT 1 FROM environment_leases l WHERE status NOT IN ('released', 'expired') - AND (${finishedFailedLease("l")}) IS NOT TRUE`, - `SELECT 1 FROM environment_custom_image_setup_sessions s WHERE + ) IS NOT TRUE`), + workCheck("environment_leases", `SELECT 1 FROM environment_leases l WHERE status NOT IN ('released', 'expired') + AND (${finishedFailedLease("l")}) IS NOT TRUE`), + workCheck("environment_custom_image_setup_sessions", `SELECT 1 FROM environment_custom_image_setup_sessions s WHERE (s.status IN ('promoted', 'cancelled', 'timed_out', 'failed') AND s.finished_at IS NOT NULL - AND s.connection_secret_ref IS NULL AND (${verifiedResourceCleanup('s', 'image_setup')})) IS NOT TRUE`, + AND s.connection_secret_ref IS NULL AND (${verifiedResourceCleanup('s', 'image_setup')})) IS NOT TRUE`), // An unconfigured Slack draft has no provider identity or inbound runtime. // Setup still wakes through normal browser admission. Retain every enabled, // previously active, partially connected, foreign, or unsupported endpoint. - `SELECT 1 FROM chat_endpoints endpoint WHERE (endpoint.provider = 'slack' + workCheck("chat_endpoints", `SELECT 1 FROM chat_endpoints endpoint WHERE (endpoint.provider = 'slack' AND endpoint.status = 'draft' AND endpoint.setup->>'step' = 'provider_setup' AND (endpoint.setup - ARRAY['step', 'command']) = '{}'::jsonb AND (endpoint.setup->'command' IS NULL OR endpoint.setup->'command' = 'null'::jsonb @@ -129,33 +133,33 @@ const WORK_CHECKS = [ AND EXISTS (SELECT 1 FROM tool_connections draft_connection WHERE draft_connection.id = endpoint.connection_id AND draft_connection.company_id = endpoint.company_id AND draft_connection.connection_purpose = 'channel' AND draft_connection.transport = 'chat_sdk' - AND draft_connection.status = 'draft' AND draft_connection.enabled = false)) IS NOT TRUE`, - `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`, - `SELECT 1 FROM routine_runs WHERE status NOT IN ('completed', 'failed', 'skipped', 'cancelled')`, - `SELECT 1 FROM plugin_jobs WHERE status <> 'paused'`, - `SELECT 1 FROM plugin_job_runs WHERE status NOT IN ('succeeded', 'failed', 'cancelled')`, - `SELECT 1 FROM plugin_webhook_deliveries WHERE status NOT IN ('succeeded', 'failed')`, + AND draft_connection.status = 'draft' AND draft_connection.enabled = false)) IS NOT TRUE`), + workCheck("routines", `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`), + workCheck("routine_runs", `SELECT 1 FROM routine_runs WHERE status NOT IN ('completed', 'failed', 'skipped', 'cancelled')`), + workCheck("plugin_jobs", `SELECT 1 FROM plugin_jobs WHERE status <> 'paused'`), + workCheck("plugin_job_runs", `SELECT 1 FROM plugin_job_runs WHERE status NOT IN ('succeeded', 'failed', 'cancelled')`), + workCheck("plugin_webhook_deliveries", `SELECT 1 FROM plugin_webhook_deliveries WHERE status NOT IN ('succeeded', 'failed')`), // Saved outbound MCP configuration and disabled drafts have no independent // listener. Keep local processes, channels and GitHub webhook bindings awake; // accepted calls, credential recovery and delivery work have separate fences. - `SELECT 1 FROM tool_connections c WHERE c.status <> 'archived' AND ( + workCheck("tool_connections", `SELECT 1 FROM tool_connections c WHERE c.status <> 'archived' AND ( (c.status = 'draft' AND c.enabled = false) OR (c.status = 'active' AND c.connection_purpose = 'ai' AND c.transport = 'runtime_auth') OR (c.status = 'active' AND c.connection_purpose = 'tool' AND c.transport = 'mcp_remote' AND NOT EXISTS (SELECT 1 FROM connection_grants g WHERE g.connection_id = c.id AND (g.company_id <> c.company_id OR g.provider_tenant->'github' IS NOT NULL))) - ) IS NOT TRUE`, + ) IS NOT TRUE`), // An explicit board escalation has no automatic retry. Retain the visible // action and all execution/cleanup checks while the board decides what to do. - `SELECT 1 FROM issue_recovery_actions a WHERE a.status NOT IN ('resolved', 'cancelled') - AND (${passiveBoardRecovery('a')}) IS NOT TRUE`, - `SELECT 1 FROM status_cards WHERE archived_at IS NULL`, - `SELECT 1 FROM external_objects WHERE NOT is_terminal OR next_refresh_at IS NOT NULL`, + workCheck("issue_recovery_actions", `SELECT 1 FROM issue_recovery_actions a WHERE a.status NOT IN ('resolved', 'cancelled') + AND (${passiveBoardRecovery('a')}) IS NOT TRUE`), + workCheck("status_cards", `SELECT 1 FROM status_cards WHERE archived_at IS NULL`), + workCheck("external_objects", `SELECT 1 FROM external_objects WHERE NOT is_terminal OR next_refresh_at IS NOT NULL`), // Login history keeps its provider reference after teardown. A terminal // label alone is insufficient: require a finished, unclaimed session and // positive cleanup evidence for its resource in the same company/environment. // Cast only UUID-shaped session references, leaving both lease indexes usable. - `SELECT 1 FROM adapter_auth_sessions s WHERE + workCheck("adapter_auth_sessions", `SELECT 1 FROM adapter_auth_sessions s WHERE s.status NOT IN ('authenticated', 'completed', 'failed', 'timed_out', 'cancelled') OR s.finished_at IS NULL OR s.promotion_expires_at IS NOT NULL OR (s.provider_lease_id IS NOT NULL AND NOT EXISTS ( @@ -174,11 +178,11 @@ const WORK_CHECKS = [ WHEN s.provider_lease_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' THEN s.provider_lease_id::uuid END) AND ((l.status IN ('released', 'expired') OR (${finishedFailedLease("l")})) AND l.cleanup_status = 'success' - AND l.released_at IS NOT NULL) IS NOT TRUE))`, + AND l.released_at IS NOT NULL) IS NOT TRUE))`), // Answer delivery history is not work once its acknowledged handoff has // finished. Keep pending deliveries, unfinished target runs, and missing or // unfinished fallback wake receipts awake. Other work checks still apply. - `SELECT 1 FROM issue_question_response_deliveries d WHERE + workCheck("issue_question_response_deliveries", `SELECT 1 FROM issue_question_response_deliveries d WHERE d.status NOT IN ('delivered', 'fallback_queued', 'failed') OR (d.status IN ('delivered', 'fallback_queued') AND d.acknowledged_at IS NULL) OR (d.status = 'delivered' AND d.target_run_id IS NULL) @@ -195,36 +199,36 @@ const WORK_CHECKS = [ OR (w.status = 'coalesced' AND EXISTS ( SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_id AND r.company_id = w.company_id AND r.agent_id = w.agent_id - AND ${finishedRun("r")})))))`, + AND ${finishedRun("r")})))))`), // Cloud owns inbound event durability. A processed receipt is idempotency // history; any downstream run, wake or delivery is still checked separately. - `SELECT 1 FROM connection_event_deliveries WHERE status <> 'processed' OR processed_at IS NULL`, + workCheck("connection_event_deliveries", `SELECT 1 FROM connection_event_deliveries WHERE status <> 'processed' OR processed_at IS NULL`), // A delivered outbox row is history after its exact accepted wake finishes. // A system-retired connection card may also be parked behind a still-pending // response. Keep the outbox intact: the normal reply path resumes delivery. - `SELECT 1 FROM connection_intent_deliveries d WHERE + workCheck("connection_intent_deliveries", `SELECT 1 FROM connection_intent_deliveries d WHERE (d.delivered_at IS NULL AND (${connectionDeliveryWaitingForResponse('d')}) IS NOT TRUE) OR EXISTS (SELECT 1 FROM agent_wakeup_requests connection_wake WHERE connection_wake.company_id = d.company_id AND connection_wake.idempotency_key IN ( 'connection-intent:' || d.interaction_id::text || ':accepted', 'connection-intent:' || d.interaction_id::text || ':rejected', 'connection-intent:' || d.interaction_id::text || ':expired') - AND (${finishedIdleWake('connection_wake')}) IS NOT TRUE)`, + AND (${finishedIdleWake('connection_wake')}) IS NOT TRUE)`), // Saved private feedback and acknowledged uploads have no delivery job. // Pending/failed exports and unfamiliar states retain the instance. A sent // label requires its completion stamp and a cleared failure reason. - `SELECT 1 FROM feedback_exports WHERE (status = 'local_only' - OR (status = 'sent' AND exported_at IS NOT NULL AND failure_reason IS NULL)) IS NOT TRUE`, + workCheck("feedback_exports", `SELECT 1 FROM feedback_exports WHERE (status = 'local_only' + OR (status = 'sent' AND exported_at IS NOT NULL AND failure_reason IS NULL)) IS NOT TRUE`), // Skill package conversion and retention are optional maintenance. Legacy // contents remain authoritative until verified conversion; retries may wait // for ingress, including when external package storage is disabled. - `SELECT 1 FROM project_skill_sources b JOIN companies c ON c.id = b.company_id - WHERE c.status = 'active' AND b.active AND b.source_id IS NULL`, - `SELECT 1 FROM company_skill_sources s JOIN companies c ON c.id = s.company_id + workCheck("project_skill_sources", `SELECT 1 FROM project_skill_sources b JOIN companies c ON c.id = b.company_id + WHERE c.status = 'active' AND b.active AND b.source_id IS NULL`), + workCheck("company_skill_sources", `SELECT 1 FROM company_skill_sources s JOIN companies c ON c.id = s.company_id WHERE c.status = 'active' AND s.enabled AND s.automatic_updates AND (s.access_scope = 'company' OR EXISTS ( - SELECT 1 FROM project_skill_sources b WHERE b.source_id = s.id AND b.company_id = s.company_id AND b.active))`, - `SELECT 1 FROM native_run_finalizations f WHERE ( + SELECT 1 FROM project_skill_sources b WHERE b.source_id = s.id AND b.company_id = s.company_id AND b.active))`), + workCheck("native_run_finalizations", `SELECT 1 FROM native_run_finalizations f WHERE ( f.phase = 'committed' AND f.lease_owner IS NULL AND f.lease_expires_at IS NULL AND f.next_attempt_at IS NULL AND f.control_deadline_at IS NULL AND f.recovery_state IS NULL AND f.recovery_request_id IS NULL @@ -258,7 +262,7 @@ const WORK_CHECKS = [ AND (effect.company_id <> f.company_id OR effect.issue_id <> f.issue_id OR effect.delivery_state <> 'delivered' OR effect.delivered_at IS NULL OR effect.next_attempt_at IS NOT NULL)) - ) IS NOT TRUE`, + ) IS NOT TRUE`), // These less common work sources fail closed on any retained state. Their // terminal-state exceptions can be added with tests for the owning service. ...[ @@ -275,7 +279,7 @@ const WORK_CHECKS = [ "dot_runner_assignments", "dot_runner_operations", "dot_mailbox_items", "muse_agent_bindings", "muse_credentials", "muse_runner_assignments", "muse_runner_operations", "muse_mailbox_items", "muse_input_deliveries", "external_agent_holds", - ].map((table) => `SELECT 1 FROM ${table}`), + ].map((table) => workCheck(table, `SELECT 1 FROM ${table}`)), ] as const; function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus, now: number): boolean { @@ -288,94 +292,166 @@ function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus /** Inventory persisted work after the task-drain admission hold lands. * Only a caller holding the exact idle owner may receive an empty report. */ +export interface IdleSleepSafetyOptions { + collectDiagnostics?: true; +} + export async function readIdleSleepSafety( db: Db, getDrainStatus: () => IdleSleepDrainStatus, now: () => number = Date.now, ownerId?: string, - inspectLocalWork: () => Promise = readIdleLocalWork, + inspectLocalWork: (diagnostic?: IdleSleepDiagnosticSink) => Promise = + diagnostic => readIdleLocalWork({ diagnostic }), inspectPlugins?: InspectIdlePlugins, prepareBackup?: () => Promise, scheduledRoutines = false, scheduledIssueMonitors = false, + options: IdleSleepSafetyOptions = {}, ): Promise { - const unknown: IdleSleepSafety = { version: 1, backgroundWork: "unknown" }; + const diagnostic = options.collectDiagnostics ? createIdleSleepDiagnostics() : undefined; + const report = (backgroundWork: IdleLocalWork, scheduledWake?: ScheduledWake): IdleSleepSafety => { + const diagnostics = diagnostic?.report(); + if (backgroundWork === "none" && diagnostics?.checks.some(check => + check.state === "unknown" || check.state === "not_checked")) backgroundWork = "unknown"; + return { version: backgroundWork === "none" && scheduledWake ? 2 : 1, backgroundWork, + ...(backgroundWork === "none" && scheduledWake ? { scheduledWake } : {}), + ...(diagnostics ? { diagnostics } : {}) }; + }; const before = getDrainStatus(); const localBefore = idleWorkSnapshot(); - if (!sameQuietHold(before, before, now())) return unknown; + const owned = () => Boolean(ownerId && ownerId === before.ownerId && before.expiresAt !== null); + if (!sameQuietHold(before, before, now()) || (diagnostic && !owned())) { + diagnostic?.record("inspection.hold", "unknown"); + diagnostic?.record("durable.snapshot", "not_checked"); + return report("unknown"); + } + diagnostic?.record("inspection.hold", "none"); + let failureReason = "local.inspection"; try { - // Reject known local blockers before querying (and potentially waking) - // the tenant database. Keep the final inspection below as a second fence. - const localFirst = await inspectLocalWork(); - if (localFirst !== "none") return { version: 1, backgroundWork: localFirst }; + const localFirst = await inspectLocalWork(diagnostic?.record); + diagnostic?.record("local.inspection", localFirst); + if (localFirst !== "none" && !diagnostic) return report(localFirst); let quietPlugins: string[] = []; + let pluginState: IdleLocalWork = "none"; if (inspectPlugins) { - if (!ownerId || ownerId !== before.ownerId || before.expiresAt === null || localBefore.active !== 0) return unknown; - const pluginWork = await inspectPlugins({ ownerId, expiresAt: before.expiresAt.getTime() }); - if (pluginWork.backgroundWork !== "none") return { version: 1, backgroundWork: pluginWork.backgroundWork }; - quietPlugins = pluginWork.pluginIds; + const currentLocal = diagnostic ? idleWorkSnapshot() : localBefore; + if (!owned() || currentLocal.active !== 0 || (diagnostic && + (!sameQuietHold(before, getDrainStatus(), now()) || currentLocal.generation !== localBefore.generation))) { + if (!diagnostic) return report("unknown"); + diagnostic.record("plugins.runtime_drain", "not_checked"); + pluginState = "unknown"; + } else { + failureReason = "plugins.runtime_drain"; + const pluginWork = await inspectPlugins({ ownerId: ownerId!, expiresAt: before.expiresAt!.getTime() }); + pluginState = pluginWork.backgroundWork; + diagnostic?.record("plugins.runtime_drain", pluginState); + if (pluginState !== "none" && !diagnostic) return report(pluginState); + if (pluginState === "none") quietPlugins = pluginWork.pluginIds; + } } + if (diagnostic && !sameQuietHold(before, getDrainStatus(), now())) { + diagnostic.record("inspection.hold", "unknown"); + diagnostic.record("durable.snapshot", "not_checked"); + return report("unknown"); + } + failureReason = "durable.snapshot"; let scheduledWake: ScheduledWake | undefined; const blocked = await db.transaction(async (tx) => { await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); - const checks = WORK_CHECKS.filter(query => - (!scheduledRoutines || query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`) - && (!scheduledIssueMonitors || query !== `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`)) - .map((query) => sql`EXISTS (${sql.raw(query)})`); - checks.push(sql`EXISTS (SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id - WHERE ${companyHoldMismatch()})`); - checks.push(sql`EXISTS (SELECT 1 FROM heartbeat_runs WHERE ${pendingBudgetRunCancellationPredicate()})`); - checks.push(sql`EXISTS (${nativeResponsePresentationPendingQuery()})`); + const checks = WORK_CHECKS.filter(check => + (!scheduledRoutines || check.reason !== "routines") + && (!scheduledIssueMonitors || check.reason !== "issues")); + checks.push(workCheck("agents.company_hold", sql`SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id + WHERE ${companyHoldMismatch()}`)); + checks.push(workCheck("heartbeat_runs.budget_cancellation", sql`SELECT 1 FROM heartbeat_runs WHERE ${pendingBudgetRunCancellationPredicate()}`)); + checks.push(workCheck("native_response_presentation", nativeResponsePresentationPendingQuery())); // Only the current worker's owned runtime drain can exempt a plugin. // Missing workers, old SDKs, and newly installed plugins still block. - checks.push(quietPlugins.length === 0 - ? sql`EXISTS (SELECT 1 FROM plugins WHERE status <> 'disabled')` - : sql`EXISTS (SELECT 1 FROM plugins WHERE status <> 'disabled' AND id NOT IN (${sql.join(quietPlugins.map((id) => sql`${id}::uuid`), sql`, `)}))`); - const rows = await tx.execute<{ blocked: boolean }>(sql`SELECT ${sql.join(checks, sql` OR `)} AS blocked`); - if (rows.length !== 1 || typeof rows[0]?.blocked !== "boolean") return undefined; - if (rows[0].blocked) return true; + checks.push(workCheck("plugins", quietPlugins.length === 0 + ? sql`SELECT 1 FROM plugins WHERE status <> 'disabled'` + : sql`SELECT 1 FROM plugins WHERE status <> 'disabled' AND id NOT IN (${sql.join(quietPlugins.map((id) => sql`${id}::uuid`), sql`, `)})`)); + let found: boolean; + if (diagnostic) { + diagnostic.expect(checks.map(check => check.reason)); + const rows = await tx.execute>(sql`SELECT ${sql.join(checks.map(check => + sql`EXISTS (${check.query}) AS ${sql.identifier(check.reason)}`), sql`, `)}`); + if (rows.length !== 1 || Object.keys(rows[0] ?? {}).length !== checks.length + || checks.some(check => !Object.hasOwn(rows[0]!, check.reason) || typeof rows[0]![check.reason] !== "boolean")) return undefined; + for (const check of checks) diagnostic.record(check.reason, rows[0]![check.reason] ? "present" : "none"); + diagnostic.record("durable.snapshot", "none"); + found = checks.some(check => rows[0]![check.reason]); + } else { + const rows = await tx.execute<{ blocked: boolean }>(sql`SELECT ${sql.join(checks.map(check => sql`EXISTS (${check.query})`), sql` OR `)} AS blocked`); + if (rows.length !== 1 || typeof rows[0]?.blocked !== "boolean") return undefined; + found = rows[0].blocked; + } + if (found && !diagnostic) return true; if (scheduledRoutines) { - if (!ownerId || ownerId !== before.ownerId) return undefined; - const schedule = await readIdleRoutineSchedule(tx, ownerId, now()); - if (schedule.blocked) return true; + if (!owned()) return undefined; + const schedule = await readIdleRoutineSchedule(tx, ownerId!, now()); + diagnostic?.record("routines.schedule", schedule.inspectionComplete === false ? "unknown" : schedule.blocked ? "present" : "none"); + if (schedule.blocked && !diagnostic) return true; + found ||= schedule.blocked && schedule.inspectionComplete !== false; scheduledWake = schedule.scheduledWake; } if (scheduledIssueMonitors) { - if (!scheduledRoutines || !ownerId || ownerId !== before.ownerId) return undefined; - const schedule = await readIdleIssueSchedule(tx, ownerId, now()); - if (schedule.blocked) return true; + if (!scheduledRoutines || !owned()) return undefined; + const schedule = await readIdleIssueSchedule(tx, ownerId!, now()); + diagnostic?.record("issues.schedule", schedule.inspectionComplete === false ? "unknown" : schedule.blocked ? "present" : "none"); + if (schedule.blocked && !diagnostic) return true; + found ||= schedule.blocked && schedule.inspectionComplete !== false; scheduledWake = combineIdleSchedules(scheduledWake, schedule.scheduledWake, schedule.generation); } - return false; + return found; }, { isolationLevel: "repeatable read", accessMode: "read only" }); - if (blocked === undefined || !sameQuietHold(before, getDrainStatus(), now())) return unknown; - if (blocked) return { version: 1, backgroundWork: "present" }; - if (!ownerId || ownerId !== before.ownerId || before.expiresAt === null || localBefore.active !== 0) return unknown; + if (blocked === undefined) { + diagnostic?.record("durable.snapshot", "unknown"); + return report("unknown"); + } + if (!sameQuietHold(before, getDrainStatus(), now())) { + diagnostic?.record("inspection.hold", "unknown"); + return report("unknown"); + } + if (blocked && !diagnostic) return report("present"); + if (!owned() || (!diagnostic && localBefore.active !== 0)) return report("unknown"); let expectedGeneration = localBefore.generation; if (prepareBackup) { - if (!sameQuietHold(before, getDrainStatus(), now()) || - idleWorkSnapshot().generation !== expectedGeneration || idleWorkSnapshot().active !== 0) return unknown; - // The runner owns exactly one tracked-work receipt through dump, - // archive verification and fsync. Its start/finish are the only work - // allowed during this scan. Any concurrent work invalidates the dump. - if (!(await prepareBackup())) return unknown; - expectedGeneration += 2; + if (diagnostic) { + diagnostic.record("backup.checkpoint", "not_checked"); + } else { + if (!sameQuietHold(before, getDrainStatus(), now()) || + idleWorkSnapshot().generation !== expectedGeneration || idleWorkSnapshot().active !== 0) return report("unknown"); + // The runner owns exactly one tracked-work receipt through dump, + // archive verification and fsync. Its start/finish are the only work + // allowed during this scan. Any concurrent work invalidates the dump. + if (!(await prepareBackup())) return report("unknown"); + expectedGeneration += 2; + } } - const local = await inspectLocalWork(); + failureReason = "local.inspection"; + const local = await inspectLocalWork(diagnostic?.record); + diagnostic?.record("local.inspection", local); const after = getDrainStatus(); const localAfter = idleWorkSnapshot(); if (!sameQuietHold(before, after, now()) || localAfter.active !== 0 || - expectedGeneration !== localAfter.generation) return unknown; - if (local === "none" && scheduledWake) { - if (Date.parse(scheduledWake.nextWakeAt) <= now()) return { version: 1, backgroundWork: "present" }; - return { version: 2, backgroundWork: "none", scheduledWake }; + expectedGeneration !== localAfter.generation) { + diagnostic?.record("inspection.generation", "unknown"); + return report("unknown"); } - return { version: 1, backgroundWork: local }; - + diagnostic?.record("inspection.generation", "none"); + if (diagnostic && (localFirst === "unknown" || local === "unknown" || pluginState === "unknown")) return report("unknown"); + if (diagnostic && (blocked || localFirst === "present" || pluginState === "present")) return report("present"); + if (local === "none" && scheduledWake && Date.parse(scheduledWake.nextWakeAt) <= now()) { + diagnostic?.record("inspection.hold", "unknown"); + return report("present"); + } + return report(local, scheduledWake); } catch { // Missing migrations, unknown state, malformed configuration, timeouts and // database failures never authorize stopping the application. Do not leak // SQL, configuration or tenant data through this control response. - return unknown; + diagnostic?.record(failureReason, "unknown"); + return report("unknown"); } }