feat(server): add domain-owned retained-history sleep proofs

Keep finished delivery and resource history while preserving its downstream
work, accounting, cleanup and transport barriers. Share the narrow waiting
conversation proof across ordinary and scheduled inspection. Retain local
runtime resources through owner reconstruction and process-local inspection.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-02 11:47:02 -06:00
1 parent 0748b90062
commit 7f454eb377
16 files changed
+1205 -21

No files matched your search

@@ -0,0 +1,49 @@
import { finishedIdleRun } from "./idle-work-proof.js";
/** Mirrors sweep/account: terminal provider status cannot settle usage. */
export function completedBrowserRun(run: string): string {
const hold = `${run}.result->'paperclipBudgetHold'`;
return `${run}.status IN ('completed', 'failed', 'cancelled')
AND ${run}.events_drained = 1 AND ${run}.recovery_cursor IS NULL
AND jsonb_typeof(${hold}) = 'object' AND ${hold}->>'invocationId' = ${run}.invocation_id::text
AND ((${run}.provider_run_id IS NOT NULL AND ${hold}->>'state' = 'settled'
AND ${hold}->>'accountedCentsExact' ~ '^[0-9]+([.][0-9]+)?$')
OR (${run}.provider_run_id IS NULL AND ${run}.status = 'failed' AND ${hold}->>'state' = 'released'))
-- account() replaces held authority with the authoritative terminal
-- summary and ledger charge. It retains prior diagnostic flags in JSON.
AND (${hold}->>'state' = 'settled' OR COALESCE(${hold}->>'billingUnknown', 'false') = 'false')
AND EXISTS (SELECT 1 FROM browser_use_sessions browser_parent_session JOIN heartbeat_runs browser_parent_run
ON browser_parent_run.id = ${run}.heartbeat_run_id AND browser_parent_run.company_id = browser_parent_session.company_id
AND browser_parent_run.agent_id = browser_parent_session.agent_id
WHERE browser_parent_session.id = ${run}.session_id AND browser_parent_session.company_id = ${run}.company_id
AND (browser_parent_run.native_issue_id = browser_parent_session.issue_id
OR browser_parent_run.context_snapshot->>'issueId' = browser_parent_session.issue_id::text
OR browser_parent_run.context_snapshot->>'taskId' = browser_parent_session.issue_id::text)
AND (browser_parent_run.native_issue_id IS NULL OR browser_parent_run.native_issue_id = browser_parent_session.issue_id)
AND (browser_parent_run.context_snapshot->>'issueId' IS NULL
OR browser_parent_run.context_snapshot->>'issueId' = browser_parent_session.issue_id::text)
AND (browser_parent_run.context_snapshot->>'taskId' IS NULL
OR browser_parent_run.context_snapshot->>'taskId' = browser_parent_session.issue_id::text)
AND ${finishedIdleRun('browser_parent_run')})`;
}
export const idleBrowserHistoryChecks = [
{ reason: "browser_use_sessions", query: `SELECT 1 FROM browser_use_sessions s WHERE (
(s.status = 'closed' OR (s.status = 'failed' AND s.provider_session_id IS NULL))
AND s.lease_token IS NULL AND s.lease_until IS NULL AND s.stop_requested IS NULL
AND EXISTS (SELECT 1 FROM issues i JOIN agents a ON a.id = s.agent_id AND a.company_id = i.company_id
JOIN tool_connections c ON c.id = s.connection_id AND c.company_id = i.company_id
JOIN connection_grants g ON g.id = s.grant_id AND g.company_id = i.company_id AND g.connection_id = c.id
WHERE i.id = s.issue_id AND i.company_id = s.company_id)
AND NOT EXISTS (SELECT 1 FROM browser_use_runs r WHERE r.session_id = s.id
AND (r.company_id <> s.company_id OR (${completedBrowserRun('r')}) IS NOT TRUE))
AND NOT EXISTS (SELECT 1 FROM browser_use_browsers b WHERE b.session_id = s.id
AND (b.company_id <> s.company_id OR b.status <> 'stopped'))) IS NOT TRUE` },
{ reason: "browser_use_runs", query: `SELECT 1 FROM browser_use_runs r WHERE (${completedBrowserRun('r')}) IS NOT TRUE` },
// stopped is written only after client.stop followed by client.browser
// confirms stopped. Parent/session/run checks remain independent barriers.
{ reason: "browser_use_browsers", query: `SELECT 1 FROM browser_use_browsers b WHERE (
b.status = 'stopped' AND EXISTS (SELECT 1 FROM browser_use_sessions s
WHERE s.id = b.session_id AND s.company_id = b.company_id AND s.status = 'closed'
AND s.stop_requested IS NULL AND s.lease_token IS NULL AND s.lease_until IS NULL)) IS NOT TRUE` },
] as const;
@@ -0,0 +1,9 @@
export function completedGithubAction(a: string): string {
return `(${a}.kind = 'github_response_comment' AND ${a}.status = 'processed' AND ${a}.result->>'final' = 'true')
OR (${a}.kind = 'github_webhook_ingress' AND (
(${a}.status = 'processed' AND ${a}.result->>'httpStatus' = '202' AND ${a}.result->>'retryable' = 'false')
OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'github_webhook_ingress_payload_invalid'
AND ${a}.result->>'retryable' = 'false')))
OR (${a}.kind = 'github_review_check' AND ${a}.status = 'processed'
AND ${a}.result->>'retryable' = 'false')`;
}
@@ -0,0 +1,79 @@
import { finishedIdleRun, finishedIdleWake } from "../idle-work-proof.js";
import { completedGithubAction } from "./github/idle-history-proof.js";
import { completedSlackAction } from "./slack/idle-history-proof.js";
import { completedTelegramAction } from "./telegram/idle-history-proof.js";
/** A processed inbound receipt can still own an unconsumed wake. */
function completedAction(a: string): string {
const wake = `EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id
AND w.id = ${a}.id AND w.agent_id::text = ${a}.payload->>'agentId'
AND w.payload->>'issueId' = ${a}.payload->>'issueId' AND ${finishedIdleWake('w')})`;
return `jsonb_typeof(${a}.payload) = 'object' AND jsonb_typeof(${a}.result) = 'object'
AND EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = ${a}.endpoint_id AND e.company_id = ${a}.company_id)
AND (${a}.delivery_id IS NULL OR EXISTS (SELECT 1 FROM chat_deliveries d
WHERE d.id = ${a}.delivery_id AND d.company_id = ${a}.company_id AND d.endpoint_id = ${a}.endpoint_id))
AND CASE
WHEN ${a}.kind = 'inbound_wakeup' THEN ${a}.status IN ('processed', 'failed') AND ${wake}
WHEN ${a}.kind = 'interaction_wakeup' THEN ${a}.status = 'processed' AND
(${a}.result->>'code' = 'interaction_wakeup_no_longer_applicable' OR EXISTS (
SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id
AND w.idempotency_key = 'interaction:' || (${a}.payload->>'interactionId') || ':' || (${a}.payload->>'interactionStatus')
AND ${finishedIdleWake('w')}))
WHEN ${a}.kind = 'failed_run_retry' THEN ${a}.status = 'processed'
AND ${a}.result->>'code' = 'failed_chat_retry_durable' AND ${wake}
WHEN ${a}.kind = 'provider_effect' THEN
(${a}.status = 'processed' AND length(${a}.result->>'providerMessageId') > 0 AND length(${a}.result->>'threadId') > 0)
OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'provider_effect_no_longer_authorized')
OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'provider_effect_failed' AND ${a}.result->>'retryable' = 'false')
WHEN ${a}.kind = 'receipt_reaction' THEN
(${a}.status = 'processed' AND jsonb_typeof(${a}.result->'attempts') = 'number' AND (${a}.result->>'attempts')::numeric > 0)
OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('receipt_reaction_superseded_by_terminal_publication', 'receipt_reaction_runtime_superseded'))
WHEN ${a}.kind = 'confirmation_response' THEN ${a}.status = 'processed'
AND ${a}.result->>'code' IN ('confirmation_resolution_committed_by_provider', 'confirmation_resolution_reconciled')
AND EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.id::text = ${a}.result->>'interactionId'
AND interaction.company_id = ${a}.company_id AND interaction.status = ${a}.result->>'interactionStatus'
AND interaction.resolved_at IS NOT NULL)
WHEN ${a}.kind = 'task_control_authorization' THEN ${a}.status IN ('processed', 'cancelled')
AND EXISTS (SELECT 1 FROM chat_publications p WHERE p.company_id = ${a}.company_id AND p.endpoint_id = ${a}.endpoint_id
AND p.conversation_id = ${a}.conversation_id AND ${a}.provider_action_id = 'task-control-authorization:' || p.id::text
AND ((p.state = 'published' AND p.published_at IS NOT NULL) OR p.state = 'cancelled'))
ELSE (${[completedGithubAction, completedSlackAction, completedTelegramAction]
.map(proof => `(${proof(a)})`).join(' OR ')}) END`;
}
export const idleChatHistoryChecks = [
{ reason: "chat_deliveries", query: `SELECT 1 FROM chat_deliveries d WHERE (
EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id)
AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL))
AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id
AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id))
AND NOT EXISTS (SELECT 1 FROM chat_actions a WHERE a.delivery_id = d.id
AND (a.company_id <> d.company_id OR a.endpoint_id <> d.endpoint_id OR (${completedAction('a')}) IS NOT TRUE))) IS NOT TRUE` },
{ reason: "chat_publications", query: `SELECT 1 FROM chat_publications p WHERE (
(p.state = 'published' AND p.published_at IS NOT NULL AND p.redacted_error IS NULL
OR (p.state IN ('failed', 'cancelled') AND p.next_attempt_at IS NULL))
AND EXISTS (SELECT 1 FROM chat_conversations c JOIN chat_endpoints e
ON e.id = c.endpoint_id AND e.company_id = c.company_id
JOIN issues i ON i.id = p.issue_id AND i.company_id = c.company_id
WHERE c.id = p.conversation_id AND c.company_id = p.company_id AND c.endpoint_id = p.endpoint_id)
AND NOT EXISTS (SELECT 1 FROM chat_endpoint_leases l WHERE l.company_id = p.company_id
AND l.endpoint_id = p.endpoint_id AND l.lease_key LIKE 'publication:' || p.id::text || ':%' AND l.expires_at > now())
AND NOT EXISTS (SELECT 1 FROM chat_teams_file_transfers f WHERE f.publication_id = p.id
AND (f.company_id <> p.company_id OR f.phase NOT IN ('delivered', 'declined', 'expired', 'cancelled')
OR f.attempt_id IS NOT NULL OR f.attempt_expires_at IS NOT NULL))) IS NOT TRUE` },
{ reason: "chat_actions", query: `SELECT 1 FROM chat_actions a WHERE (${completedAction('a')}) IS NOT TRUE` },
{ reason: "chat_completion_deliveries", query: `SELECT 1 FROM chat_completion_deliveries d WHERE (
d.status IN ('delivered', 'superseded', 'exhausted')
AND EXISTS (SELECT 1 FROM chat_task_handoffs h JOIN issues task ON task.id = h.task_id AND task.company_id = h.company_id
JOIN issues source ON source.id = h.conversation_id AND source.company_id = h.company_id
WHERE h.task_id = d.task_id AND h.company_id = d.company_id
AND source.conversation_agent_id = h.agent_id
AND (d.status <> 'delivered' OR (d.status_version = task.status_version AND task.status = 'done'
AND EXISTS (SELECT 1 FROM issue_comments c WHERE c.id = d.response_comment_id
AND c.company_id = d.company_id AND c.issue_id = h.conversation_id AND c.author_agent_id = h.agent_id))))
AND (d.target_run_id IS NULL AND d.status <> 'delivered' OR EXISTS (
SELECT 1 FROM heartbeat_runs r JOIN chat_task_handoffs h ON h.task_id = d.task_id
WHERE r.id = d.target_run_id AND r.company_id = d.company_id AND r.agent_id = h.agent_id
AND (r.native_issue_id = h.conversation_id OR r.context_snapshot->>'issueId' = h.conversation_id::text)
AND ${finishedIdleRun('r')}))) IS NOT TRUE` },
] as const;
@@ -0,0 +1,16 @@
export function completedSlackAction(a: string): string {
return `(${a}.kind = 'slack_session_sync' AND (
(${a}.status = 'processed' AND ${a}.result->>'outcome' IN ('updated', 'unsupported')
AND (${a}.result->>'outcome' <> 'updated' OR ${a}.result->>'sessionStatus' <> 'processing'))
OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('slack_session_no_published_state', 'slack_session_destination_changed'))
OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'slack_session_sync_rejected')))
OR (${a}.kind = 'slash_task_start' AND ${a}.status = 'processed'
AND length(${a}.result->>'providerMessageId') > 0 AND length(${a}.result->>'threadId') > 0)
OR (${a}.kind = 'slack_file_upload_receipt' AND ${a}.status = 'processed'
AND ${a}.result->>'code' = 'slack_file_upload_identity_confirmed')
OR (${a}.kind = 'slack_session_stop' AND ${a}.status IN ('processed', 'cancelled')
AND ${a}.result->>'code' IN ('slack_session_stop_cancelled', 'slack_session_stop_target_superseded', 'slack_session_stop_no_current_work')
AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id::text = ${a}.result->>'runId'
AND (r.company_id <> ${a}.company_id OR r.status NOT IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')
OR r.finished_at IS NULL OR r.cost_accounting_pending)))`;
}
@@ -0,0 +1,12 @@
export function completedTelegramAction(a: string): string {
return `(${a}.kind = 'telegram_maintenance' AND (
(${a}.status = 'processed' AND (${a}.result->>'providerConfirmed' = 'true'
OR ${a}.result->>'code' = 'telegram_stop_subscription_confirmed'))
OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'telegram_maintenance_credentials_superseded')
OR (${a}.status = 'failed' AND ${a}.result->>'retryable' = 'false'
AND ${a}.result->>'code' = 'telegram_maintenance_failed')))
OR (${a}.kind = 'telegram_draft' AND (
(${a}.status = 'processed' AND ${a}.result->>'phase' = 'published' AND length(${a}.result->>'providerMessageId') > 0)
OR (${a}.status = 'cancelled' AND (${a}.result->>'phase' = 'stopped'
OR ${a}.result->>'code' = 'telegram_draft_presentation_stopped'))))`;
}
@@ -0,0 +1,45 @@
import { finishedIdleRun, finishedIdleWake } from "./idle-work-proof.js";
/** The same positive container proof is used by ordinary and scheduled sleep.
* waiting/in_review is set by settleConversationTurn, but is not sufficient:
* a user comment may commit before the wake is enqueued. */
export function passiveIdleConversation(i: "i"): string {
return `${i}.conversation_agent_id IS NOT NULL AND length(${i}.conversation_user_id) > 0
AND ${i}.conversation_state = 'waiting' AND ${i}.status = 'in_review'
AND ${i}.assignee_agent_id = ${i}.conversation_agent_id AND ${i}.assignee_user_id IS NULL
AND (${i}.conversation_user_id = 'local-board' OR EXISTS (SELECT 1 FROM company_memberships membership
WHERE membership.company_id = ${i}.company_id AND membership.principal_type = 'user'
AND membership.principal_id = ${i}.conversation_user_id AND membership.status = 'active'))
AND ${i}.conversation_session_generation >= 0
AND ${i}.checkout_run_id IS NULL AND ${i}.execution_run_id IS NULL
AND ${i}.monitor_next_check_at IS NULL AND ${i}.monitor_wake_requested_at IS NULL
AND (${i}.execution_policy IS NULL OR (jsonb_typeof(${i}.execution_policy) = 'object'
AND COALESCE(${i}.execution_policy->'monitor', 'null'::jsonb) = 'null'::jsonb))
AND (${i}.execution_state IS NULL OR (jsonb_typeof(${i}.execution_state) = 'object'
AND ${i}.execution_state->>'status' = 'idle'
AND COALESCE(${i}.execution_state->'monitor', 'null'::jsonb) = 'null'::jsonb
AND COALESCE(${i}.execution_state->'currentParticipant', 'null'::jsonb) = 'null'::jsonb
AND COALESCE(${i}.execution_state->'currentStageId', 'null'::jsonb) = 'null'::jsonb
AND COALESCE(${i}.execution_state->'reviewRequest', 'null'::jsonb) = 'null'::jsonb))
AND EXISTS (SELECT 1 FROM agents a JOIN companies c ON c.id = a.company_id
WHERE a.id = ${i}.conversation_agent_id AND a.company_id = ${i}.company_id
AND c.status = 'active' AND a.status IN ('idle', 'active', 'paused', 'terminated'))
AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE
(r.native_issue_id = ${i}.id OR r.context_snapshot->>'issueId' = ${i}.id::text OR r.context_snapshot->>'taskId' = ${i}.id::text)
AND (r.company_id <> ${i}.company_id OR r.agent_id <> ${i}.conversation_agent_id OR (${finishedIdleRun('r')}) IS NOT TRUE))
AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE
(w.payload->>'issueId' = ${i}.id::text OR w.payload->>'taskId' = ${i}.id::text)
AND (w.company_id <> ${i}.company_id OR w.agent_id <> ${i}.conversation_agent_id
OR (${finishedIdleWake('w')}) IS NOT TRUE))
AND NOT EXISTS (SELECT 1 FROM issue_comments comment WHERE comment.issue_id = ${i}.id
AND comment.deleted_at IS NULL AND comment.author_user_id IS NOT NULL AND comment.author_agent_id IS NULL
AND comment.origin = 'comment' AND comment.client_request_id IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${i}.company_id
AND w.agent_id = ${i}.conversation_agent_id AND w.idempotency_key = 'conversation-comment:' || comment.id::text
AND ${finishedIdleWake('w')}))
AND NOT EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.issue_id = ${i}.id
AND (interaction.company_id <> ${i}.company_id OR interaction.status = 'pending'))
AND NOT EXISTS (SELECT 1 FROM chat_completion_deliveries d JOIN chat_task_handoffs h ON h.task_id = d.task_id
WHERE h.conversation_id = ${i}.id AND (h.company_id <> ${i}.company_id OR d.company_id <> ${i}.company_id
OR d.status NOT IN ('delivered', 'superseded', 'exhausted')))`;
}
+134
View File
@@ -0,0 +1,134 @@
import { finishedIdleRun } from "./idle-work-proof.js";
/** executeDecisionEffects visits every selected ordinal, including failures.
* Its final aggregate cannot prove a missing row or another option's effect. */
function completedEffectManifest(d: string): string {
const options = `jsonb_array_elements(CASE WHEN jsonb_typeof(${d}.options) = 'array' THEN ${d}.options ELSE '[]'::jsonb END)`;
return `EXISTS (SELECT 1 FROM ${options} option_row WHERE option_row->>'id' = ${d}.chosen_option_id
AND jsonb_typeof(option_row->'effects') = 'array'
AND EXISTS (SELECT 1 FROM activity_log selection WHERE selection.company_id = ${d}.company_id
AND selection.entity_type = 'decision' AND selection.entity_id = ${d}.id::text
AND selection.action IN ('decision.decided', 'decision.dismissed')
AND selection.details->>'chosenOptionId' = ${d}.chosen_option_id
AND selection.details->>'decidedByUserId' = ${d}.decided_by_user_id)
AND (SELECT count(*) FROM ${options} duplicate_option WHERE duplicate_option->>'id' = ${d}.chosen_option_id) = 1
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(option_row->'effects') WITH ORDINALITY expected(effect, ordinal)
WHERE NOT EXISTS (SELECT 1 FROM decision_effect_executions execution JOIN issues target
ON target.id = execution.target_issue_id AND target.company_id = ${d}.company_id
JOIN activity_log audit ON audit.id = execution.activity_log_id AND audit.company_id = ${d}.company_id
AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text
AND audit.details->>'chosenOptionId' = ${d}.chosen_option_id
AND audit.details->>'executionId' = execution.id::text
WHERE execution.decision_id = ${d}.id AND execution.effect_index = expected.ordinal - 1
AND execution.effect_type = expected.effect->>'type' AND execution.target_issue_id::text = expected.effect->>'targetIssueId'
AND execution.status IN ('executed', 'failed', 'skipped') AND execution.executed_at IS NOT NULL
AND audit.action = 'decision.effect_' || execution.status))
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions execution WHERE execution.decision_id = ${d}.id
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(option_row->'effects') WITH ORDINALITY expected(effect, ordinal)
WHERE execution.effect_index = expected.ordinal - 1 AND execution.effect_type = expected.effect->>'type'
AND execution.target_issue_id::text = expected.effect->>'targetIssueId'))
AND CASE WHEN ${d}.execution_status = 'succeeded' THEN NOT EXISTS (SELECT 1 FROM decision_effect_executions e
WHERE e.decision_id = ${d}.id AND e.status <> 'executed')
WHEN ${d}.execution_status = 'partial' THEN EXISTS (SELECT 1 FROM decision_effect_executions e
WHERE e.decision_id = ${d}.id AND e.status = 'executed') AND EXISTS (SELECT 1 FROM decision_effect_executions e
WHERE e.decision_id = ${d}.id AND e.status IN ('failed', 'skipped'))
ELSE NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id AND e.status = 'executed')
AND EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id AND e.status IN ('failed', 'skipped')) END)`;
}
function completedDecisionOutcome(d: string): string {
return `(${d}.status IN ('expired', 'cancelled') AND ${d}.execution_status IS NULL AND ${d}.chosen_option_id IS NULL
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id)
AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id
AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.' || ${d}.status))
OR (${d}.status = 'decided' AND ${d}.decided_at IS NOT NULL AND ${d}.execution_status IN ('succeeded', 'partial', 'failed')
AND ((COALESCE(${d}.metadata->>'kind', '') <> 'attention_archive_proposal' AND (${completedEffectManifest(d)}))
OR (${completedArchiveOutcome(d)})
OR (${d}.chosen_option_id = 'dismissed' AND ${d}.execution_status = 'succeeded' AND ${d}.metadata->>'dismissed' = 'true'
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id)
AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id
AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.dismissed'
AND audit.details->>'chosenOptionId' = 'dismissed' AND audit.details->>'decidedByUserId' = ${d}.decided_by_user_id))
OR (${d}.metadata->>'effectRecoveryError' IN ('missing_decider', 'missing_option')
AND ${d}.execution_status IN ('failed', 'partial')
AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id
AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.recovery_failed'
AND audit.details->>'reason' = ${d}.metadata->>'effectRecoveryError'
AND audit.details->>'executionStatus' = ${d}.execution_status
AND audit.details->>'chosenOptionId' IS NOT DISTINCT FROM ${d}.chosen_option_id))))`;
}
/** Archive applies the reviewed set in one transaction. Every catch is final
* (execution_status=failed); it neither replays the set nor leaves per-effect
* claims. Continuation and notification queues remain independent checks. */
function completedArchiveOutcome(d: string): string {
return `${d}.metadata->>'kind' = 'attention_archive_proposal'
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id)
AND EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(${d}.options) = 'array'
THEN ${d}.options ELSE '[]'::jsonb END) selected WHERE selected->>'id' = ${d}.chosen_option_id
AND selected->'effects' = '[]'::jsonb
AND (SELECT count(*) FROM jsonb_array_elements(${d}.options) duplicate WHERE duplicate->>'id' = ${d}.chosen_option_id) = 1)
AND ((${d}.execution_status = 'failed' AND ${d}.chosen_option_id = 'archive'
AND jsonb_typeof(${d}.metadata->'archiveProposalError') = 'string'
AND length(trim(${d}.metadata->>'archiveProposalError')) > 0)
OR (${d}.execution_status = 'succeeded' AND ${d}.metadata->>'archiveProposalError' IS NULL
AND (${d}.chosen_option_id <> 'archive' OR (
jsonb_typeof(${d}.target_snapshots) = 'object' AND ${d}.target_snapshots <> '{}'::jsonb
AND ${d}.metadata->>'manifestHash' ~ '^[a-f0-9]{64}$'
AND NOT EXISTS (SELECT 1 FROM jsonb_each(${d}.target_snapshots) snapshot WHERE (
snapshot.key = 'attention:' || (snapshot.value->'attentionArchive'->>'sourceKind') || ':' || (snapshot.value->'attentionArchive'->>'sourceId')
AND snapshot.value->'attentionArchive'->>'companyId' = ${d}.company_id::text
AND jsonb_typeof(snapshot.value->'attentionArchive'->'expectedVersion') = 'number'
AND snapshot.value->'attentionArchive'->>'expectedVersion' ~ '^[1-9][0-9]*$'
AND EXISTS (SELECT 1 FROM decision_retention retention WHERE retention.company_id = ${d}.company_id
AND retention.source_kind = snapshot.value->'attentionArchive'->>'sourceKind'
AND retention.source_id = snapshot.value->'attentionArchive'->>'sourceId'
AND retention.version::numeric > (snapshot.value->'attentionArchive'->>'expectedVersion')::numeric)
AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id
AND audit.actor_id = 'decision-archive-proposal' AND audit.action = 'decision_retention.archived'
AND audit.entity_type = 'attention_source' AND audit.entity_id = snapshot.value->'attentionArchive'->>'sourceId'
AND audit.details->>'sourceKind' = snapshot.value->'attentionArchive'->>'sourceKind'
AND audit.created_at >= ${d}.decided_at AND audit.responsible_user_id = ${d}.decided_by_user_id)) IS NOT TRUE)))))`;
}
function completedDecision(alias: string): string {
return `(${completedDecisionOutcome(alias)})
AND ${alias}.continuation_policy IN ('none', 'wake_origin_agent')
AND jsonb_typeof(${alias}.metadata) = 'object'
AND COALESCE(${alias}.metadata->>'continuationPending', 'false') = 'false'
AND (${alias}.continuation_policy = 'none' OR ${alias}.metadata->>'continuationDeliveredAt' IS NOT NULL)
AND EXISTS (SELECT 1 FROM heartbeat_runs r JOIN issues i ON i.id = ${alias}.origin_issue_id
AND i.company_id = r.company_id WHERE r.id = ${alias}.origin_run_id
AND r.company_id = ${alias}.company_id AND r.agent_id = ${alias}.origin_agent_id AND ${finishedIdleRun('r')})
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${alias}.id
AND (e.status NOT IN ('executed', 'failed', 'skipped') OR e.executed_at IS NULL))`;
}
export const idleDecisionHistoryChecks = [
{ reason: "decisions", query: `SELECT 1 FROM decisions d WHERE (${completedDecision('d')}) IS NOT TRUE` },
{ reason: "decision_effect_executions", query: `SELECT 1 FROM decision_effect_executions e WHERE (
e.status IN ('executed', 'failed', 'skipped') AND e.executed_at IS NOT NULL
AND EXISTS (SELECT 1 FROM decisions d JOIN issues i ON i.id = e.target_issue_id AND i.company_id = d.company_id
WHERE d.id = e.decision_id AND (${completedDecision('d')}))) IS NOT TRUE` },
{ reason: "decision_archive_notification_outbox", query: `SELECT 1 FROM decision_archive_notification_outbox o WHERE (
o.status = 'delivered' AND o.delivered_at IS NOT NULL
AND o.archive_version > 0 AND EXISTS (SELECT 1 FROM decision_retention retention
WHERE retention.company_id = o.company_id AND retention.source_kind = o.source_kind
AND retention.source_id = o.source_id AND retention.archive_version >= o.archive_version)
AND EXISTS (SELECT 1 FROM agents a JOIN issues i ON i.company_id = a.company_id
WHERE a.id = o.origin_agent_id AND i.id = o.origin_issue_id AND i.company_id = o.company_id)) IS NOT TRUE` },
{ reason: "company_secret_proposals", query: `SELECT 1 FROM company_secret_proposals p WHERE (
p.kind IN ('secret', 'binding') AND p.status IN ('approved', 'rejected', 'withdrawn', 'expired')
AND p.resolved_at IS NOT NULL AND p.value_ciphertext IS NULL AND p.ciphertext_scrubbed_at IS NOT NULL
AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = p.origin_run_id
AND r.company_id = p.company_id AND r.agent_id = p.proposed_by_agent_id AND ${finishedIdleRun('r')})
AND (p.origin_issue_id IS NULL OR EXISTS (SELECT 1 FROM issues i
WHERE i.id = p.origin_issue_id AND i.company_id = p.company_id))
AND (p.status <> 'approved' OR (p.kind = 'secret' AND EXISTS (SELECT 1 FROM company_secrets s
WHERE s.id = p.created_secret_id AND s.company_id = p.company_id))
OR (p.kind = 'binding' AND p.applied_binding_config_path = p.config_path AND EXISTS (
SELECT 1 FROM agents a WHERE a.id = p.target_id AND a.company_id = p.company_id)))
AND (p.interaction_id IS NULL OR EXISTS (SELECT 1 FROM issue_thread_interactions i
WHERE i.id = p.interaction_id AND i.company_id = p.company_id AND i.issue_id = p.origin_issue_id
AND i.status IN ('accepted', 'rejected', 'cancelled', 'expired') AND i.resolved_at IS NOT NULL))) IS NOT TRUE` },
] as const;
+5 -2
View File
@@ -5,6 +5,7 @@ import { AGENT_STATUSES, issueExecutionMonitorPolicySchema, PROVIDER_QUOTA_MONIT
import type { ScheduledWake } from "./idle-scheduled-work.js";
import { onboardingFirstTaskAwaitingUserSql } from "./recovery/onboarding-first-task-wait.js";
import { completedAgentUnblockRequest, responseDrivenIssueWait } from "./idle-work-proof.js";
import { passiveIdleConversation } from "./conversation-idle-proof.js";
const processGeneration = randomUUID();
const limit = 10_000;
@@ -30,6 +31,7 @@ export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: st
untouched_onboarding: boolean; response_driven_wait: boolean;
unblock_request_finished: boolean;
response_wait_generation: unknown;
passive_conversation: boolean;
}>(sql`SELECT i.id, i.company_id, i.status, i.updated_at, i.assignee_agent_id, i.assignee_user_id,
i.checkout_run_id, i.execution_run_id, i.execution_policy, i.execution_state, i.unblock_descriptor,
i.conversation_agent_id, i.monitor_next_check_at, i.monitor_wake_requested_at, i.monitor_attempt_count,
@@ -38,6 +40,7 @@ export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: st
(i.assignee_agent_id IS NULL OR a.id IS NOT NULL) AS scoped,
(${sql.raw(responseDrivenIssueWait('i'))}) AS response_driven_wait,
(${sql.raw(completedAgentUnblockRequest('i'))}) AS unblock_request_finished,
(${sql.raw(passiveIdleConversation('i'))}) AS passive_conversation,
(SELECT jsonb_agg(jsonb_build_array(w.id, w.updated_at, w.kind, w.status, w.source_run_id,
w.effective_resolver_policy, w.continuation_policy, w.addressee_agent_id, w.addressee_user_id) ORDER BY w.id)
FROM issue_thread_interactions w WHERE w.issue_id = i.id AND w.status = 'pending') AS response_wait_generation,
@@ -61,7 +64,7 @@ export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: st
if (!statuses.has(row.status) || row.company_status !== "active" || row.scoped !== true
|| (row.assignee_agent_id && !agentStatuses.has(row.agent_status ?? ""))
|| row.unsafe_dependencies !== false || row.checkout_run_id || row.execution_run_id
|| row.conversation_agent_id || row.monitor_wake_requested_at
|| (row.conversation_agent_id && row.passive_conversation !== true) || row.monitor_wake_requested_at
|| (row.execution_state !== null && state?.status !== "idle")
|| (row.execution_policy !== null && !policy)) return { blocked: true };
if (row.status === "blocked" && row.dependency_ready) return { blocked: true };
@@ -81,7 +84,7 @@ export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: st
}
if (policy?.monitor != null) return { blocked: true };
const unblock = record(row.unblock_descriptor);
const passive = row.status === "backlog"
const passive = row.passive_conversation === true || row.status === "backlog"
|| (row.status === "todo" && !row.assignee_agent_id)
|| (row.status === "todo" && row.assignee_agent_id && row.agent_status === "idle" && row.untouched_onboarding === true)
|| (row.status === "blocked" && (row.unblock_descriptor === null || unblock?.owner === "board"
+20 -19
View File
@@ -9,6 +9,23 @@ import { verifiedResourceCleanup } from "./cleanup-verification-proof.js";
import { companyHoldMismatch } from "../modules/agent-lifecycle/index.js";
import { pendingBudgetRunCancellationPredicate } from "./heartbeat/budget-cancellation.js";
import { nativeResponsePresentationPendingQuery } from "./native-runtime/native-response-presentation-query.js";
import { passiveIdleConversation } from "./conversation-idle-proof.js";
import { idleChatHistoryChecks } from "./chat-channels/idle-history-proof.js";
import { idleToolActionHistoryChecks } from "./tool-action-idle-proof.js";
import { idleBrowserHistoryChecks } from "./browser-use-idle-proof.js";
import { idleDecisionHistoryChecks } from "./decision-idle-proof.js";
import { idleWorkspaceHistoryChecks } from "./workspace-history-idle-proof.js";
import { idleMcpHistoryChecks } from "./public-mcp/idle-history-proof.js";
import { idleExternalAgentHistoryChecks } from "../modules/external-agents/idle-history-proof.js";
export const completedHistoryChecks = [
...idleChatHistoryChecks, ...idleToolActionHistoryChecks, ...idleBrowserHistoryChecks,
...idleDecisionHistoryChecks, ...idleWorkspaceHistoryChecks, ...idleMcpHistoryChecks,
...idleExternalAgentHistoryChecks,
] as const;
const ordinaryIssues = `SELECT 1 FROM issues i WHERE i.status NOT IN ('done', 'cancelled')
AND (${passiveIdleConversation('i')}) IS NOT TRUE`;
export interface IdleSleepDrainStatus {
ownerId?: string;
@@ -74,7 +91,7 @@ const WORK_CHECKS = [
SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_id
AND r.company_id = w.company_id AND r.agent_id = w.agent_id
AND ${finishedRun("r")})))`,
`SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`,
ordinaryIssues,
// A completed issue can still need its first watchdog review, including a
// retry after immediate evaluation failed before creating a review/run.
`SELECT 1 FROM issue_watchdogs WHERE status <> 'disabled'`,
@@ -259,23 +276,7 @@ const WORK_CHECKS = [
OR effect.delivery_state <> 'delivered' OR effect.delivered_at IS NULL
OR effect.next_attempt_at IS NOT NULL))
) IS NOT TRUE`,
// These less common work sources fail closed on any retained state. Their
// terminal-state exceptions can be added with tests for the owning service.
...[
"chat_deliveries", "chat_publications", "chat_actions",
"chat_completion_deliveries", "tool_action_requests", "tool_action_deliveries",
"workspace_runtime_services",
"execution_workspace_runtime_leases",
"company_transfer_runs", "decisions", "decision_effect_executions",
"decision_archive_notification_outbox", "browser_use_sessions", "browser_use_runs",
"browser_use_browsers",
"company_secret_proposals",
"mcp_oauth_grants", "mcp_mutation_receipts", "mcp_event_subscriptions",
"mcp_event_deliveries", "mcp_attachment_uploads", "dot_agent_bindings",
"dot_runner_assignments", "dot_runner_operations", "dot_mailbox_items",
"muse_agent_bindings", "muse_credentials", "muse_runner_assignments",
"muse_runner_operations", "muse_mailbox_items", "muse_input_deliveries", "external_agent_holds",
].map((table) => `SELECT 1 FROM ${table}`),
...completedHistoryChecks.map(check => check.query),
] as const;
function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus, now: number): boolean {
@@ -320,7 +321,7 @@ export async function readIdleSleepSafety(
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
const checks = WORK_CHECKS.filter(query =>
(!scheduledRoutines || query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`)
&& (!scheduledIssueMonitors || query !== `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`))
&& (!scheduledIssueMonitors || query !== ordinaryIssues))
.map((query) => sql`EXISTS (${sql.raw(query)})`);
checks.push(sql`EXISTS (SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id
WHERE ${companyHoldMismatch()})`);
@@ -0,0 +1,37 @@
/** Completion receipts are retained for idempotency. Fixed aliases only. */
export const idleMcpHistoryChecks = [
{ reason: "mcp_oauth_grants", query: `SELECT 1 FROM mcp_oauth_grants g WHERE
(g.revoked_at IS NOT NULL AND EXISTS (SELECT 1 FROM companies c WHERE c.id = g.company_id)) IS NOT TRUE` },
{ reason: "mcp_mutation_receipts", query: `SELECT 1 FROM mcp_mutation_receipts m WHERE (
m.status = 'completed' AND jsonb_typeof(m.result) = 'object'
AND EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = m.grant_id
AND g.company_id = m.company_id AND g.user_id = m.user_id)) IS NOT TRUE` },
// Stopped subscriptions no longer scan for events. Expiry alone is not a
// handoff of responsibility for active subscriptions or unfinished delivery.
{ reason: "mcp_event_subscriptions", query: `SELECT 1 FROM mcp_event_subscriptions s WHERE (
s.stopped_at IS NOT NULL AND EXISTS (SELECT 1 FROM mcp_oauth_grants g
WHERE g.id = s.grant_id AND g.company_id = s.company_id)) IS NOT TRUE` },
{ reason: "mcp_event_deliveries", query: `SELECT 1 FROM mcp_event_deliveries d WHERE (
d.finished_at IS NOT NULL AND (d.outcome IN ('delivered', 'filtered', 'inactive', 'delivery_exhausted')
OR (d.outcome ~ '^http_[1-5][0-9][0-9]$' AND d.outcome !~ '^http_2'
AND (d.attempts >= 6 OR (d.outcome NOT IN ('http_408', 'http_429') AND d.outcome !~ '^http_5'))))
AND EXISTS (SELECT 1 FROM mcp_event_subscriptions s WHERE s.id = d.subscription_id
AND EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = s.grant_id AND g.company_id = s.company_id)
AND ((d.activity_id IS NOT NULL AND d.mailbox_item_id IS NULL AND EXISTS (
SELECT 1 FROM activity_log a WHERE a.id = d.activity_id AND a.company_id = s.company_id))
OR (d.mailbox_item_id IS NOT NULL AND d.activity_id IS NULL AND EXISTS (
SELECT 1 FROM dot_mailbox_items m WHERE m.id = d.mailbox_item_id
AND m.company_id = s.company_id AND m.binding_id = s.binding_id))))) IS NOT TRUE` },
// The upload owner allocates object identity before storage writes and only
// stamps cleaned_at after deletion. A committed attachment is never deleted.
{ reason: "mcp_attachment_uploads", query: `SELECT 1 FROM mcp_attachment_uploads u WHERE (
EXISTS (SELECT 1 FROM issues i WHERE i.id = u.task_id AND i.company_id = u.company_id)
AND (u.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = u.grant_id
AND g.company_id = u.company_id AND g.user_id = u.user_id))
AND ((u.attachment_id IS NULL AND u.cleaned_at IS NOT NULL)
OR (u.cleaned_at IS NULL AND u.attachment_id IS NOT NULL AND EXISTS (
SELECT 1 FROM issue_attachments a JOIN assets stored ON stored.id = a.asset_id
WHERE a.id = u.attachment_id AND a.company_id = u.company_id AND a.issue_id = u.task_id
AND stored.company_id = u.company_id AND stored.object_key = u.object_key
AND stored.provider = u.storage_provider)))) IS NOT TRUE` },
] as const;
@@ -0,0 +1,28 @@
import { finishedIdleRun } from "./idle-work-proof.js";
function completedRequest(a: string): string {
return `${a}.status IN ('executed', 'failed', 'rejected', 'expired', 'cancelled') AND ${a}.resolved_at IS NOT NULL
AND EXISTS (SELECT 1 FROM tool_invocations v WHERE v.id = ${a}.invocation_id AND v.company_id = ${a}.company_id
AND v.issue_id IS NOT DISTINCT FROM ${a}.issue_id AND v.completed_at IS NOT NULL
AND v.status IN ('succeeded', 'failed', 'denied', 'cancelled', 'timed_out', 'rate_limited')
AND (v.run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = v.run_id
AND r.company_id = v.company_id AND ${finishedIdleRun('r')})))
AND (${a}.interaction_id IS NULL OR EXISTS (SELECT 1 FROM issue_thread_interactions i WHERE i.id = ${a}.interaction_id
AND i.company_id = ${a}.company_id AND i.issue_id = ${a}.issue_id AND i.resolved_at IS NOT NULL
AND i.payload->'toolAction'->>'actionRequestId' = ${a}.id::text
AND i.payload->'toolAction'->>'invocationId' = ${a}.invocation_id::text
AND i.status IN ('accepted', 'rejected', 'expired', 'cancelled')
AND (${a}.status = 'rejected' OR i.result->'toolAction'->>'status' = CASE
WHEN ${a}.status = 'cancelled' THEN 'expired' ELSE ${a}.status END)))`;
}
export const idleToolActionHistoryChecks = [
{ reason: "tool_action_requests", query: `SELECT 1 FROM tool_action_requests a WHERE (${completedRequest('a')}) IS NOT TRUE` },
// delivered_at is the domain's acknowledgment. It also retires outcomes
// after issue completion/reassignment. Accepted continuations stay covered by
// the global wake/run checks, including coalesced run accounting.
{ reason: "tool_action_deliveries", query: `SELECT 1 FROM tool_action_deliveries d WHERE (
d.delivered_at IS NOT NULL AND EXISTS (SELECT 1 FROM tool_action_requests a
WHERE a.id = d.action_request_id AND a.company_id = d.company_id
AND a.issue_id = d.issue_id AND a.interaction_id = d.interaction_id AND (${completedRequest('a')}))) IS NOT TRUE` },
] as const;
@@ -0,0 +1,37 @@
import { finishedIdleRun } from "./idle-work-proof.js";
export const idleWorkspaceHistoryChecks = [
// Failed imports are intentionally resumable. Completed/cancelled rows can
// still owe spool deletion, independently inspected by idle-local-work.
{ reason: "company_transfer_runs", query: `SELECT 1 FROM company_transfer_runs t WHERE (
t.status IN ('completed', 'cancelled') AND t.finished_at IS NOT NULL
AND t.direction IN ('import', 'export') AND jsonb_typeof(t.container_ref) = 'object'
AND jsonb_typeof(t.completed_parts) = 'array'
AND (t.company_id IS NOT NULL OR (t.direction = 'import' AND t.status = 'cancelled'))
AND (t.company_id IS NULL OR EXISTS (SELECT 1 FROM companies c WHERE c.id = t.company_id))) IS NOT TRUE` },
{ reason: "workspace_runtime_services", query: `SELECT 1 FROM workspace_runtime_services s WHERE (
s.status = 'stopped' AND s.stopped_at IS NOT NULL AND s.provider = 'local_process'
AND s.exposure_handle IS NULL
AND (s.exposure IS NULL OR (jsonb_typeof(s.exposure) = 'object'
AND s.exposure->>'state' = 'removed' AND s.url IS NULL
AND COALESCE(s.exposure->'publicUrl', 'null'::jsonb) = 'null'::jsonb))
AND (s.execution_workspace_id IS NULL OR EXISTS (SELECT 1 FROM execution_workspaces w
WHERE w.id = s.execution_workspace_id AND w.company_id = s.company_id))
AND (s.issue_id IS NULL OR EXISTS (SELECT 1 FROM issues i WHERE i.id = s.issue_id AND i.company_id = s.company_id))
AND (s.started_by_run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r
WHERE r.id = s.started_by_run_id AND r.company_id = s.company_id AND ${finishedIdleRun('r')}))) IS NOT TRUE` },
// This is an exclusivity row, not a provider lease. The domain only
// evaluates staleness on the next claim; no expiry job is owed by history.
{ reason: "execution_workspace_runtime_leases", query: `SELECT 1 FROM execution_workspace_runtime_leases l WHERE (
EXISTS (SELECT 1 FROM execution_workspaces w WHERE w.id = l.execution_workspace_id AND w.company_id = l.company_id)
AND ((l.owner_issue_id IS NOT NULL AND l.owner_key = 'issue:' || l.owner_issue_id::text
AND EXISTS (SELECT 1 FROM issues i WHERE i.id = l.owner_issue_id AND i.company_id = l.company_id
AND i.status IN ('done', 'cancelled')))
OR (l.owner_issue_id IS NULL AND l.owner_run_id IS NOT NULL AND l.owner_key = 'run:' || l.owner_run_id::text
AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = l.owner_run_id AND r.company_id = l.company_id
AND r.agent_id = l.owner_agent_id AND ${finishedIdleRun('r')})))
AND (l.owner_run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = l.owner_run_id
AND r.company_id = l.company_id AND r.agent_id = l.owner_agent_id AND ${finishedIdleRun('r')}))
AND NOT EXISTS (SELECT 1 FROM workspace_runtime_services s WHERE s.execution_workspace_id = l.execution_workspace_id
AND (s.company_id <> l.company_id OR s.status <> 'stopped' OR s.stopped_at IS NULL))) IS NOT TRUE` },
] as const;
+32
View File
@@ -46,6 +46,7 @@ import {
import { and, desc, eq, gte, inArray, isNull, lte, ne, or, sql } from "drizzle-orm";
import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js";
import { conflict, unprocessable } from "../errors.js";
import { registerIdleLocalWorkProbe, type IdleLocalWork } from "./idle-local-work.js";
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
import { resolveHomeAwarePath } from "../home-paths.js";
import { hasVerifiedWorktreeSeedManifest, isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
@@ -283,6 +284,22 @@ const quarantinedRuntimeExposurePorts = new Set<number>();
* companion together or not at all.
*/
const exposurePortPairClaims = new ExposurePortPairClaims();
let runtimeReconstructionComplete = false;
let runtimeReconstructionEpoch = 0;
let runtimeReconciliation: ReturnType<typeof reconstructRuntimeServices> | null = null;
/** Read owner memory only. Durable stopped rows do not describe a surviving
* registry process, an allocation being published, or unfinished exposure cleanup. */
export function readWorkspaceRuntimeIdleWork(): IdleLocalWork {
if (!runtimeReconstructionComplete || runtimeReconciliation !== null) return "unknown";
return runtimeServicesById.size || runtimeProvisionByWorkspace.size || runtimeControlStartByOwner.size
|| runtimeReplacementClaimsByReuseKey.size || quarantinedRuntimeExposurePorts.size
|| exposurePortPairClaims.activePorts().size
|| [...inFlightAllocatedPorts.values()].some(until => until > Date.now()) ? "present" : "none";
}
// This registry and its probe share the application process lifetime, like the
// native-session owner. Stopping one resource must not unregister its siblings.
registerIdleLocalWorkProbe(readWorkspaceRuntimeIdleWork);
/**
* Execution-workspace statuses that still hold an exclusive lease. `archived`
* is the only terminal state; everything else — including `idle` — is a lane an
@@ -525,6 +542,8 @@ type ProcessOutputAccumulator = {
export async function resetRuntimeServicesForTests(
opts: { terminateProcesses?: boolean; simulateSupervisorExit?: boolean } = {},
) {
runtimeReconstructionComplete = false;
runtimeReconstructionEpoch += 1;
if (opts.terminateProcesses) {
for (const serviceId of [...runtimeServicesById.keys()]) {
await stopRuntimeService(serviceId).catch(() => undefined);
@@ -8577,6 +8596,19 @@ export async function refreshPersistedRuntimeServiceHealth(input: {
}
export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
if (runtimeReconciliation) return runtimeReconciliation;
runtimeReconstructionComplete = false;
const epoch = ++runtimeReconstructionEpoch;
runtimeReconciliation = reconstructRuntimeServices(db).then(result => {
if (epoch === runtimeReconstructionEpoch) {
runtimeReconstructionComplete = result.restartFailed === 0 && result.exposureReservationDrift.length === 0;
}
return result;
}).finally(() => { runtimeReconciliation = null; });
return runtimeReconciliation;
}
async function reconstructRuntimeServices(db: Db) {
const rows = await db
.select()
.from(workspaceRuntimeServices)