mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 05:56:19 +01:00
feat(server): add domain-owned retained-history sleep proofs
Keep finished delivery and resource history while preserving its downstream work, accounting, cleanup and transport barriers. Share the narrow waiting conversation proof across ordinary and scheduled inspection. Retain local runtime resources through owner reconstruction and process-local inspection. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
0748b90062
commit
7f454eb377
16 files changed
+1205
-21
No files matched your search
@@ -0,0 +1,49 @@
|
||||
import { finishedIdleRun } from "./idle-work-proof.js";
|
||||
|
||||
/** Mirrors sweep/account: terminal provider status cannot settle usage. */
|
||||
export function completedBrowserRun(run: string): string {
|
||||
const hold = `${run}.result->'paperclipBudgetHold'`;
|
||||
return `${run}.status IN ('completed', 'failed', 'cancelled')
|
||||
AND ${run}.events_drained = 1 AND ${run}.recovery_cursor IS NULL
|
||||
AND jsonb_typeof(${hold}) = 'object' AND ${hold}->>'invocationId' = ${run}.invocation_id::text
|
||||
AND ((${run}.provider_run_id IS NOT NULL AND ${hold}->>'state' = 'settled'
|
||||
AND ${hold}->>'accountedCentsExact' ~ '^[0-9]+([.][0-9]+)?$')
|
||||
OR (${run}.provider_run_id IS NULL AND ${run}.status = 'failed' AND ${hold}->>'state' = 'released'))
|
||||
-- account() replaces held authority with the authoritative terminal
|
||||
-- summary and ledger charge. It retains prior diagnostic flags in JSON.
|
||||
AND (${hold}->>'state' = 'settled' OR COALESCE(${hold}->>'billingUnknown', 'false') = 'false')
|
||||
AND EXISTS (SELECT 1 FROM browser_use_sessions browser_parent_session JOIN heartbeat_runs browser_parent_run
|
||||
ON browser_parent_run.id = ${run}.heartbeat_run_id AND browser_parent_run.company_id = browser_parent_session.company_id
|
||||
AND browser_parent_run.agent_id = browser_parent_session.agent_id
|
||||
WHERE browser_parent_session.id = ${run}.session_id AND browser_parent_session.company_id = ${run}.company_id
|
||||
AND (browser_parent_run.native_issue_id = browser_parent_session.issue_id
|
||||
OR browser_parent_run.context_snapshot->>'issueId' = browser_parent_session.issue_id::text
|
||||
OR browser_parent_run.context_snapshot->>'taskId' = browser_parent_session.issue_id::text)
|
||||
AND (browser_parent_run.native_issue_id IS NULL OR browser_parent_run.native_issue_id = browser_parent_session.issue_id)
|
||||
AND (browser_parent_run.context_snapshot->>'issueId' IS NULL
|
||||
OR browser_parent_run.context_snapshot->>'issueId' = browser_parent_session.issue_id::text)
|
||||
AND (browser_parent_run.context_snapshot->>'taskId' IS NULL
|
||||
OR browser_parent_run.context_snapshot->>'taskId' = browser_parent_session.issue_id::text)
|
||||
AND ${finishedIdleRun('browser_parent_run')})`;
|
||||
}
|
||||
|
||||
export const idleBrowserHistoryChecks = [
|
||||
{ reason: "browser_use_sessions", query: `SELECT 1 FROM browser_use_sessions s WHERE (
|
||||
(s.status = 'closed' OR (s.status = 'failed' AND s.provider_session_id IS NULL))
|
||||
AND s.lease_token IS NULL AND s.lease_until IS NULL AND s.stop_requested IS NULL
|
||||
AND EXISTS (SELECT 1 FROM issues i JOIN agents a ON a.id = s.agent_id AND a.company_id = i.company_id
|
||||
JOIN tool_connections c ON c.id = s.connection_id AND c.company_id = i.company_id
|
||||
JOIN connection_grants g ON g.id = s.grant_id AND g.company_id = i.company_id AND g.connection_id = c.id
|
||||
WHERE i.id = s.issue_id AND i.company_id = s.company_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM browser_use_runs r WHERE r.session_id = s.id
|
||||
AND (r.company_id <> s.company_id OR (${completedBrowserRun('r')}) IS NOT TRUE))
|
||||
AND NOT EXISTS (SELECT 1 FROM browser_use_browsers b WHERE b.session_id = s.id
|
||||
AND (b.company_id <> s.company_id OR b.status <> 'stopped'))) IS NOT TRUE` },
|
||||
{ reason: "browser_use_runs", query: `SELECT 1 FROM browser_use_runs r WHERE (${completedBrowserRun('r')}) IS NOT TRUE` },
|
||||
// stopped is written only after client.stop followed by client.browser
|
||||
// confirms stopped. Parent/session/run checks remain independent barriers.
|
||||
{ reason: "browser_use_browsers", query: `SELECT 1 FROM browser_use_browsers b WHERE (
|
||||
b.status = 'stopped' AND EXISTS (SELECT 1 FROM browser_use_sessions s
|
||||
WHERE s.id = b.session_id AND s.company_id = b.company_id AND s.status = 'closed'
|
||||
AND s.stop_requested IS NULL AND s.lease_token IS NULL AND s.lease_until IS NULL)) IS NOT TRUE` },
|
||||
] as const;
|
||||
@@ -0,0 +1,9 @@
|
||||
export function completedGithubAction(a: string): string {
|
||||
return `(${a}.kind = 'github_response_comment' AND ${a}.status = 'processed' AND ${a}.result->>'final' = 'true')
|
||||
OR (${a}.kind = 'github_webhook_ingress' AND (
|
||||
(${a}.status = 'processed' AND ${a}.result->>'httpStatus' = '202' AND ${a}.result->>'retryable' = 'false')
|
||||
OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'github_webhook_ingress_payload_invalid'
|
||||
AND ${a}.result->>'retryable' = 'false')))
|
||||
OR (${a}.kind = 'github_review_check' AND ${a}.status = 'processed'
|
||||
AND ${a}.result->>'retryable' = 'false')`;
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { finishedIdleRun, finishedIdleWake } from "../idle-work-proof.js";
|
||||
import { completedGithubAction } from "./github/idle-history-proof.js";
|
||||
import { completedSlackAction } from "./slack/idle-history-proof.js";
|
||||
import { completedTelegramAction } from "./telegram/idle-history-proof.js";
|
||||
|
||||
/** A processed inbound receipt can still own an unconsumed wake. */
|
||||
function completedAction(a: string): string {
|
||||
const wake = `EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id
|
||||
AND w.id = ${a}.id AND w.agent_id::text = ${a}.payload->>'agentId'
|
||||
AND w.payload->>'issueId' = ${a}.payload->>'issueId' AND ${finishedIdleWake('w')})`;
|
||||
return `jsonb_typeof(${a}.payload) = 'object' AND jsonb_typeof(${a}.result) = 'object'
|
||||
AND EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = ${a}.endpoint_id AND e.company_id = ${a}.company_id)
|
||||
AND (${a}.delivery_id IS NULL OR EXISTS (SELECT 1 FROM chat_deliveries d
|
||||
WHERE d.id = ${a}.delivery_id AND d.company_id = ${a}.company_id AND d.endpoint_id = ${a}.endpoint_id))
|
||||
AND CASE
|
||||
WHEN ${a}.kind = 'inbound_wakeup' THEN ${a}.status IN ('processed', 'failed') AND ${wake}
|
||||
WHEN ${a}.kind = 'interaction_wakeup' THEN ${a}.status = 'processed' AND
|
||||
(${a}.result->>'code' = 'interaction_wakeup_no_longer_applicable' OR EXISTS (
|
||||
SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id
|
||||
AND w.idempotency_key = 'interaction:' || (${a}.payload->>'interactionId') || ':' || (${a}.payload->>'interactionStatus')
|
||||
AND ${finishedIdleWake('w')}))
|
||||
WHEN ${a}.kind = 'failed_run_retry' THEN ${a}.status = 'processed'
|
||||
AND ${a}.result->>'code' = 'failed_chat_retry_durable' AND ${wake}
|
||||
WHEN ${a}.kind = 'provider_effect' THEN
|
||||
(${a}.status = 'processed' AND length(${a}.result->>'providerMessageId') > 0 AND length(${a}.result->>'threadId') > 0)
|
||||
OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'provider_effect_no_longer_authorized')
|
||||
OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'provider_effect_failed' AND ${a}.result->>'retryable' = 'false')
|
||||
WHEN ${a}.kind = 'receipt_reaction' THEN
|
||||
(${a}.status = 'processed' AND jsonb_typeof(${a}.result->'attempts') = 'number' AND (${a}.result->>'attempts')::numeric > 0)
|
||||
OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('receipt_reaction_superseded_by_terminal_publication', 'receipt_reaction_runtime_superseded'))
|
||||
WHEN ${a}.kind = 'confirmation_response' THEN ${a}.status = 'processed'
|
||||
AND ${a}.result->>'code' IN ('confirmation_resolution_committed_by_provider', 'confirmation_resolution_reconciled')
|
||||
AND EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.id::text = ${a}.result->>'interactionId'
|
||||
AND interaction.company_id = ${a}.company_id AND interaction.status = ${a}.result->>'interactionStatus'
|
||||
AND interaction.resolved_at IS NOT NULL)
|
||||
WHEN ${a}.kind = 'task_control_authorization' THEN ${a}.status IN ('processed', 'cancelled')
|
||||
AND EXISTS (SELECT 1 FROM chat_publications p WHERE p.company_id = ${a}.company_id AND p.endpoint_id = ${a}.endpoint_id
|
||||
AND p.conversation_id = ${a}.conversation_id AND ${a}.provider_action_id = 'task-control-authorization:' || p.id::text
|
||||
AND ((p.state = 'published' AND p.published_at IS NOT NULL) OR p.state = 'cancelled'))
|
||||
ELSE (${[completedGithubAction, completedSlackAction, completedTelegramAction]
|
||||
.map(proof => `(${proof(a)})`).join(' OR ')}) END`;
|
||||
}
|
||||
|
||||
export const idleChatHistoryChecks = [
|
||||
{ reason: "chat_deliveries", query: `SELECT 1 FROM chat_deliveries d WHERE (
|
||||
EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id)
|
||||
AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL))
|
||||
AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id
|
||||
AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id))
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_actions a WHERE a.delivery_id = d.id
|
||||
AND (a.company_id <> d.company_id OR a.endpoint_id <> d.endpoint_id OR (${completedAction('a')}) IS NOT TRUE))) IS NOT TRUE` },
|
||||
{ reason: "chat_publications", query: `SELECT 1 FROM chat_publications p WHERE (
|
||||
(p.state = 'published' AND p.published_at IS NOT NULL AND p.redacted_error IS NULL
|
||||
OR (p.state IN ('failed', 'cancelled') AND p.next_attempt_at IS NULL))
|
||||
AND EXISTS (SELECT 1 FROM chat_conversations c JOIN chat_endpoints e
|
||||
ON e.id = c.endpoint_id AND e.company_id = c.company_id
|
||||
JOIN issues i ON i.id = p.issue_id AND i.company_id = c.company_id
|
||||
WHERE c.id = p.conversation_id AND c.company_id = p.company_id AND c.endpoint_id = p.endpoint_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_endpoint_leases l WHERE l.company_id = p.company_id
|
||||
AND l.endpoint_id = p.endpoint_id AND l.lease_key LIKE 'publication:' || p.id::text || ':%' AND l.expires_at > now())
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_teams_file_transfers f WHERE f.publication_id = p.id
|
||||
AND (f.company_id <> p.company_id OR f.phase NOT IN ('delivered', 'declined', 'expired', 'cancelled')
|
||||
OR f.attempt_id IS NOT NULL OR f.attempt_expires_at IS NOT NULL))) IS NOT TRUE` },
|
||||
{ reason: "chat_actions", query: `SELECT 1 FROM chat_actions a WHERE (${completedAction('a')}) IS NOT TRUE` },
|
||||
{ reason: "chat_completion_deliveries", query: `SELECT 1 FROM chat_completion_deliveries d WHERE (
|
||||
d.status IN ('delivered', 'superseded', 'exhausted')
|
||||
AND EXISTS (SELECT 1 FROM chat_task_handoffs h JOIN issues task ON task.id = h.task_id AND task.company_id = h.company_id
|
||||
JOIN issues source ON source.id = h.conversation_id AND source.company_id = h.company_id
|
||||
WHERE h.task_id = d.task_id AND h.company_id = d.company_id
|
||||
AND source.conversation_agent_id = h.agent_id
|
||||
AND (d.status <> 'delivered' OR (d.status_version = task.status_version AND task.status = 'done'
|
||||
AND EXISTS (SELECT 1 FROM issue_comments c WHERE c.id = d.response_comment_id
|
||||
AND c.company_id = d.company_id AND c.issue_id = h.conversation_id AND c.author_agent_id = h.agent_id))))
|
||||
AND (d.target_run_id IS NULL AND d.status <> 'delivered' OR EXISTS (
|
||||
SELECT 1 FROM heartbeat_runs r JOIN chat_task_handoffs h ON h.task_id = d.task_id
|
||||
WHERE r.id = d.target_run_id AND r.company_id = d.company_id AND r.agent_id = h.agent_id
|
||||
AND (r.native_issue_id = h.conversation_id OR r.context_snapshot->>'issueId' = h.conversation_id::text)
|
||||
AND ${finishedIdleRun('r')}))) IS NOT TRUE` },
|
||||
] as const;
|
||||
@@ -0,0 +1,16 @@
|
||||
export function completedSlackAction(a: string): string {
|
||||
return `(${a}.kind = 'slack_session_sync' AND (
|
||||
(${a}.status = 'processed' AND ${a}.result->>'outcome' IN ('updated', 'unsupported')
|
||||
AND (${a}.result->>'outcome' <> 'updated' OR ${a}.result->>'sessionStatus' <> 'processing'))
|
||||
OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('slack_session_no_published_state', 'slack_session_destination_changed'))
|
||||
OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'slack_session_sync_rejected')))
|
||||
OR (${a}.kind = 'slash_task_start' AND ${a}.status = 'processed'
|
||||
AND length(${a}.result->>'providerMessageId') > 0 AND length(${a}.result->>'threadId') > 0)
|
||||
OR (${a}.kind = 'slack_file_upload_receipt' AND ${a}.status = 'processed'
|
||||
AND ${a}.result->>'code' = 'slack_file_upload_identity_confirmed')
|
||||
OR (${a}.kind = 'slack_session_stop' AND ${a}.status IN ('processed', 'cancelled')
|
||||
AND ${a}.result->>'code' IN ('slack_session_stop_cancelled', 'slack_session_stop_target_superseded', 'slack_session_stop_no_current_work')
|
||||
AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id::text = ${a}.result->>'runId'
|
||||
AND (r.company_id <> ${a}.company_id OR r.status NOT IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')
|
||||
OR r.finished_at IS NULL OR r.cost_accounting_pending)))`;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
export function completedTelegramAction(a: string): string {
|
||||
return `(${a}.kind = 'telegram_maintenance' AND (
|
||||
(${a}.status = 'processed' AND (${a}.result->>'providerConfirmed' = 'true'
|
||||
OR ${a}.result->>'code' = 'telegram_stop_subscription_confirmed'))
|
||||
OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'telegram_maintenance_credentials_superseded')
|
||||
OR (${a}.status = 'failed' AND ${a}.result->>'retryable' = 'false'
|
||||
AND ${a}.result->>'code' = 'telegram_maintenance_failed')))
|
||||
OR (${a}.kind = 'telegram_draft' AND (
|
||||
(${a}.status = 'processed' AND ${a}.result->>'phase' = 'published' AND length(${a}.result->>'providerMessageId') > 0)
|
||||
OR (${a}.status = 'cancelled' AND (${a}.result->>'phase' = 'stopped'
|
||||
OR ${a}.result->>'code' = 'telegram_draft_presentation_stopped'))))`;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { finishedIdleRun, finishedIdleWake } from "./idle-work-proof.js";
|
||||
|
||||
/** The same positive container proof is used by ordinary and scheduled sleep.
|
||||
* waiting/in_review is set by settleConversationTurn, but is not sufficient:
|
||||
* a user comment may commit before the wake is enqueued. */
|
||||
export function passiveIdleConversation(i: "i"): string {
|
||||
return `${i}.conversation_agent_id IS NOT NULL AND length(${i}.conversation_user_id) > 0
|
||||
AND ${i}.conversation_state = 'waiting' AND ${i}.status = 'in_review'
|
||||
AND ${i}.assignee_agent_id = ${i}.conversation_agent_id AND ${i}.assignee_user_id IS NULL
|
||||
AND (${i}.conversation_user_id = 'local-board' OR EXISTS (SELECT 1 FROM company_memberships membership
|
||||
WHERE membership.company_id = ${i}.company_id AND membership.principal_type = 'user'
|
||||
AND membership.principal_id = ${i}.conversation_user_id AND membership.status = 'active'))
|
||||
AND ${i}.conversation_session_generation >= 0
|
||||
AND ${i}.checkout_run_id IS NULL AND ${i}.execution_run_id IS NULL
|
||||
AND ${i}.monitor_next_check_at IS NULL AND ${i}.monitor_wake_requested_at IS NULL
|
||||
AND (${i}.execution_policy IS NULL OR (jsonb_typeof(${i}.execution_policy) = 'object'
|
||||
AND COALESCE(${i}.execution_policy->'monitor', 'null'::jsonb) = 'null'::jsonb))
|
||||
AND (${i}.execution_state IS NULL OR (jsonb_typeof(${i}.execution_state) = 'object'
|
||||
AND ${i}.execution_state->>'status' = 'idle'
|
||||
AND COALESCE(${i}.execution_state->'monitor', 'null'::jsonb) = 'null'::jsonb
|
||||
AND COALESCE(${i}.execution_state->'currentParticipant', 'null'::jsonb) = 'null'::jsonb
|
||||
AND COALESCE(${i}.execution_state->'currentStageId', 'null'::jsonb) = 'null'::jsonb
|
||||
AND COALESCE(${i}.execution_state->'reviewRequest', 'null'::jsonb) = 'null'::jsonb))
|
||||
AND EXISTS (SELECT 1 FROM agents a JOIN companies c ON c.id = a.company_id
|
||||
WHERE a.id = ${i}.conversation_agent_id AND a.company_id = ${i}.company_id
|
||||
AND c.status = 'active' AND a.status IN ('idle', 'active', 'paused', 'terminated'))
|
||||
AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE
|
||||
(r.native_issue_id = ${i}.id OR r.context_snapshot->>'issueId' = ${i}.id::text OR r.context_snapshot->>'taskId' = ${i}.id::text)
|
||||
AND (r.company_id <> ${i}.company_id OR r.agent_id <> ${i}.conversation_agent_id OR (${finishedIdleRun('r')}) IS NOT TRUE))
|
||||
AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE
|
||||
(w.payload->>'issueId' = ${i}.id::text OR w.payload->>'taskId' = ${i}.id::text)
|
||||
AND (w.company_id <> ${i}.company_id OR w.agent_id <> ${i}.conversation_agent_id
|
||||
OR (${finishedIdleWake('w')}) IS NOT TRUE))
|
||||
AND NOT EXISTS (SELECT 1 FROM issue_comments comment WHERE comment.issue_id = ${i}.id
|
||||
AND comment.deleted_at IS NULL AND comment.author_user_id IS NOT NULL AND comment.author_agent_id IS NULL
|
||||
AND comment.origin = 'comment' AND comment.client_request_id IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${i}.company_id
|
||||
AND w.agent_id = ${i}.conversation_agent_id AND w.idempotency_key = 'conversation-comment:' || comment.id::text
|
||||
AND ${finishedIdleWake('w')}))
|
||||
AND NOT EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.issue_id = ${i}.id
|
||||
AND (interaction.company_id <> ${i}.company_id OR interaction.status = 'pending'))
|
||||
AND NOT EXISTS (SELECT 1 FROM chat_completion_deliveries d JOIN chat_task_handoffs h ON h.task_id = d.task_id
|
||||
WHERE h.conversation_id = ${i}.id AND (h.company_id <> ${i}.company_id OR d.company_id <> ${i}.company_id
|
||||
OR d.status NOT IN ('delivered', 'superseded', 'exhausted')))`;
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import { finishedIdleRun } from "./idle-work-proof.js";
|
||||
|
||||
/** executeDecisionEffects visits every selected ordinal, including failures.
|
||||
* Its final aggregate cannot prove a missing row or another option's effect. */
|
||||
function completedEffectManifest(d: string): string {
|
||||
const options = `jsonb_array_elements(CASE WHEN jsonb_typeof(${d}.options) = 'array' THEN ${d}.options ELSE '[]'::jsonb END)`;
|
||||
return `EXISTS (SELECT 1 FROM ${options} option_row WHERE option_row->>'id' = ${d}.chosen_option_id
|
||||
AND jsonb_typeof(option_row->'effects') = 'array'
|
||||
AND EXISTS (SELECT 1 FROM activity_log selection WHERE selection.company_id = ${d}.company_id
|
||||
AND selection.entity_type = 'decision' AND selection.entity_id = ${d}.id::text
|
||||
AND selection.action IN ('decision.decided', 'decision.dismissed')
|
||||
AND selection.details->>'chosenOptionId' = ${d}.chosen_option_id
|
||||
AND selection.details->>'decidedByUserId' = ${d}.decided_by_user_id)
|
||||
AND (SELECT count(*) FROM ${options} duplicate_option WHERE duplicate_option->>'id' = ${d}.chosen_option_id) = 1
|
||||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(option_row->'effects') WITH ORDINALITY expected(effect, ordinal)
|
||||
WHERE NOT EXISTS (SELECT 1 FROM decision_effect_executions execution JOIN issues target
|
||||
ON target.id = execution.target_issue_id AND target.company_id = ${d}.company_id
|
||||
JOIN activity_log audit ON audit.id = execution.activity_log_id AND audit.company_id = ${d}.company_id
|
||||
AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text
|
||||
AND audit.details->>'chosenOptionId' = ${d}.chosen_option_id
|
||||
AND audit.details->>'executionId' = execution.id::text
|
||||
WHERE execution.decision_id = ${d}.id AND execution.effect_index = expected.ordinal - 1
|
||||
AND execution.effect_type = expected.effect->>'type' AND execution.target_issue_id::text = expected.effect->>'targetIssueId'
|
||||
AND execution.status IN ('executed', 'failed', 'skipped') AND execution.executed_at IS NOT NULL
|
||||
AND audit.action = 'decision.effect_' || execution.status))
|
||||
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions execution WHERE execution.decision_id = ${d}.id
|
||||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(option_row->'effects') WITH ORDINALITY expected(effect, ordinal)
|
||||
WHERE execution.effect_index = expected.ordinal - 1 AND execution.effect_type = expected.effect->>'type'
|
||||
AND execution.target_issue_id::text = expected.effect->>'targetIssueId'))
|
||||
AND CASE WHEN ${d}.execution_status = 'succeeded' THEN NOT EXISTS (SELECT 1 FROM decision_effect_executions e
|
||||
WHERE e.decision_id = ${d}.id AND e.status <> 'executed')
|
||||
WHEN ${d}.execution_status = 'partial' THEN EXISTS (SELECT 1 FROM decision_effect_executions e
|
||||
WHERE e.decision_id = ${d}.id AND e.status = 'executed') AND EXISTS (SELECT 1 FROM decision_effect_executions e
|
||||
WHERE e.decision_id = ${d}.id AND e.status IN ('failed', 'skipped'))
|
||||
ELSE NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id AND e.status = 'executed')
|
||||
AND EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id AND e.status IN ('failed', 'skipped')) END)`;
|
||||
}
|
||||
|
||||
function completedDecisionOutcome(d: string): string {
|
||||
return `(${d}.status IN ('expired', 'cancelled') AND ${d}.execution_status IS NULL AND ${d}.chosen_option_id IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id)
|
||||
AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id
|
||||
AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.' || ${d}.status))
|
||||
OR (${d}.status = 'decided' AND ${d}.decided_at IS NOT NULL AND ${d}.execution_status IN ('succeeded', 'partial', 'failed')
|
||||
AND ((COALESCE(${d}.metadata->>'kind', '') <> 'attention_archive_proposal' AND (${completedEffectManifest(d)}))
|
||||
OR (${completedArchiveOutcome(d)})
|
||||
OR (${d}.chosen_option_id = 'dismissed' AND ${d}.execution_status = 'succeeded' AND ${d}.metadata->>'dismissed' = 'true'
|
||||
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id)
|
||||
AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id
|
||||
AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.dismissed'
|
||||
AND audit.details->>'chosenOptionId' = 'dismissed' AND audit.details->>'decidedByUserId' = ${d}.decided_by_user_id))
|
||||
OR (${d}.metadata->>'effectRecoveryError' IN ('missing_decider', 'missing_option')
|
||||
AND ${d}.execution_status IN ('failed', 'partial')
|
||||
AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id
|
||||
AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.recovery_failed'
|
||||
AND audit.details->>'reason' = ${d}.metadata->>'effectRecoveryError'
|
||||
AND audit.details->>'executionStatus' = ${d}.execution_status
|
||||
AND audit.details->>'chosenOptionId' IS NOT DISTINCT FROM ${d}.chosen_option_id))))`;
|
||||
}
|
||||
|
||||
/** Archive applies the reviewed set in one transaction. Every catch is final
|
||||
* (execution_status=failed); it neither replays the set nor leaves per-effect
|
||||
* claims. Continuation and notification queues remain independent checks. */
|
||||
function completedArchiveOutcome(d: string): string {
|
||||
return `${d}.metadata->>'kind' = 'attention_archive_proposal'
|
||||
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id)
|
||||
AND EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(${d}.options) = 'array'
|
||||
THEN ${d}.options ELSE '[]'::jsonb END) selected WHERE selected->>'id' = ${d}.chosen_option_id
|
||||
AND selected->'effects' = '[]'::jsonb
|
||||
AND (SELECT count(*) FROM jsonb_array_elements(${d}.options) duplicate WHERE duplicate->>'id' = ${d}.chosen_option_id) = 1)
|
||||
AND ((${d}.execution_status = 'failed' AND ${d}.chosen_option_id = 'archive'
|
||||
AND jsonb_typeof(${d}.metadata->'archiveProposalError') = 'string'
|
||||
AND length(trim(${d}.metadata->>'archiveProposalError')) > 0)
|
||||
OR (${d}.execution_status = 'succeeded' AND ${d}.metadata->>'archiveProposalError' IS NULL
|
||||
AND (${d}.chosen_option_id <> 'archive' OR (
|
||||
jsonb_typeof(${d}.target_snapshots) = 'object' AND ${d}.target_snapshots <> '{}'::jsonb
|
||||
AND ${d}.metadata->>'manifestHash' ~ '^[a-f0-9]{64}$'
|
||||
AND NOT EXISTS (SELECT 1 FROM jsonb_each(${d}.target_snapshots) snapshot WHERE (
|
||||
snapshot.key = 'attention:' || (snapshot.value->'attentionArchive'->>'sourceKind') || ':' || (snapshot.value->'attentionArchive'->>'sourceId')
|
||||
AND snapshot.value->'attentionArchive'->>'companyId' = ${d}.company_id::text
|
||||
AND jsonb_typeof(snapshot.value->'attentionArchive'->'expectedVersion') = 'number'
|
||||
AND snapshot.value->'attentionArchive'->>'expectedVersion' ~ '^[1-9][0-9]*$'
|
||||
AND EXISTS (SELECT 1 FROM decision_retention retention WHERE retention.company_id = ${d}.company_id
|
||||
AND retention.source_kind = snapshot.value->'attentionArchive'->>'sourceKind'
|
||||
AND retention.source_id = snapshot.value->'attentionArchive'->>'sourceId'
|
||||
AND retention.version::numeric > (snapshot.value->'attentionArchive'->>'expectedVersion')::numeric)
|
||||
AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id
|
||||
AND audit.actor_id = 'decision-archive-proposal' AND audit.action = 'decision_retention.archived'
|
||||
AND audit.entity_type = 'attention_source' AND audit.entity_id = snapshot.value->'attentionArchive'->>'sourceId'
|
||||
AND audit.details->>'sourceKind' = snapshot.value->'attentionArchive'->>'sourceKind'
|
||||
AND audit.created_at >= ${d}.decided_at AND audit.responsible_user_id = ${d}.decided_by_user_id)) IS NOT TRUE)))))`;
|
||||
}
|
||||
|
||||
function completedDecision(alias: string): string {
|
||||
return `(${completedDecisionOutcome(alias)})
|
||||
AND ${alias}.continuation_policy IN ('none', 'wake_origin_agent')
|
||||
AND jsonb_typeof(${alias}.metadata) = 'object'
|
||||
AND COALESCE(${alias}.metadata->>'continuationPending', 'false') = 'false'
|
||||
AND (${alias}.continuation_policy = 'none' OR ${alias}.metadata->>'continuationDeliveredAt' IS NOT NULL)
|
||||
AND EXISTS (SELECT 1 FROM heartbeat_runs r JOIN issues i ON i.id = ${alias}.origin_issue_id
|
||||
AND i.company_id = r.company_id WHERE r.id = ${alias}.origin_run_id
|
||||
AND r.company_id = ${alias}.company_id AND r.agent_id = ${alias}.origin_agent_id AND ${finishedIdleRun('r')})
|
||||
AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${alias}.id
|
||||
AND (e.status NOT IN ('executed', 'failed', 'skipped') OR e.executed_at IS NULL))`;
|
||||
}
|
||||
|
||||
export const idleDecisionHistoryChecks = [
|
||||
{ reason: "decisions", query: `SELECT 1 FROM decisions d WHERE (${completedDecision('d')}) IS NOT TRUE` },
|
||||
{ reason: "decision_effect_executions", query: `SELECT 1 FROM decision_effect_executions e WHERE (
|
||||
e.status IN ('executed', 'failed', 'skipped') AND e.executed_at IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM decisions d JOIN issues i ON i.id = e.target_issue_id AND i.company_id = d.company_id
|
||||
WHERE d.id = e.decision_id AND (${completedDecision('d')}))) IS NOT TRUE` },
|
||||
{ reason: "decision_archive_notification_outbox", query: `SELECT 1 FROM decision_archive_notification_outbox o WHERE (
|
||||
o.status = 'delivered' AND o.delivered_at IS NOT NULL
|
||||
AND o.archive_version > 0 AND EXISTS (SELECT 1 FROM decision_retention retention
|
||||
WHERE retention.company_id = o.company_id AND retention.source_kind = o.source_kind
|
||||
AND retention.source_id = o.source_id AND retention.archive_version >= o.archive_version)
|
||||
AND EXISTS (SELECT 1 FROM agents a JOIN issues i ON i.company_id = a.company_id
|
||||
WHERE a.id = o.origin_agent_id AND i.id = o.origin_issue_id AND i.company_id = o.company_id)) IS NOT TRUE` },
|
||||
{ reason: "company_secret_proposals", query: `SELECT 1 FROM company_secret_proposals p WHERE (
|
||||
p.kind IN ('secret', 'binding') AND p.status IN ('approved', 'rejected', 'withdrawn', 'expired')
|
||||
AND p.resolved_at IS NOT NULL AND p.value_ciphertext IS NULL AND p.ciphertext_scrubbed_at IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = p.origin_run_id
|
||||
AND r.company_id = p.company_id AND r.agent_id = p.proposed_by_agent_id AND ${finishedIdleRun('r')})
|
||||
AND (p.origin_issue_id IS NULL OR EXISTS (SELECT 1 FROM issues i
|
||||
WHERE i.id = p.origin_issue_id AND i.company_id = p.company_id))
|
||||
AND (p.status <> 'approved' OR (p.kind = 'secret' AND EXISTS (SELECT 1 FROM company_secrets s
|
||||
WHERE s.id = p.created_secret_id AND s.company_id = p.company_id))
|
||||
OR (p.kind = 'binding' AND p.applied_binding_config_path = p.config_path AND EXISTS (
|
||||
SELECT 1 FROM agents a WHERE a.id = p.target_id AND a.company_id = p.company_id)))
|
||||
AND (p.interaction_id IS NULL OR EXISTS (SELECT 1 FROM issue_thread_interactions i
|
||||
WHERE i.id = p.interaction_id AND i.company_id = p.company_id AND i.issue_id = p.origin_issue_id
|
||||
AND i.status IN ('accepted', 'rejected', 'cancelled', 'expired') AND i.resolved_at IS NOT NULL))) IS NOT TRUE` },
|
||||
] as const;
|
||||
@@ -5,6 +5,7 @@ import { AGENT_STATUSES, issueExecutionMonitorPolicySchema, PROVIDER_QUOTA_MONIT
|
||||
import type { ScheduledWake } from "./idle-scheduled-work.js";
|
||||
import { onboardingFirstTaskAwaitingUserSql } from "./recovery/onboarding-first-task-wait.js";
|
||||
import { completedAgentUnblockRequest, responseDrivenIssueWait } from "./idle-work-proof.js";
|
||||
import { passiveIdleConversation } from "./conversation-idle-proof.js";
|
||||
|
||||
const processGeneration = randomUUID();
|
||||
const limit = 10_000;
|
||||
@@ -30,6 +31,7 @@ export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: st
|
||||
untouched_onboarding: boolean; response_driven_wait: boolean;
|
||||
unblock_request_finished: boolean;
|
||||
response_wait_generation: unknown;
|
||||
passive_conversation: boolean;
|
||||
}>(sql`SELECT i.id, i.company_id, i.status, i.updated_at, i.assignee_agent_id, i.assignee_user_id,
|
||||
i.checkout_run_id, i.execution_run_id, i.execution_policy, i.execution_state, i.unblock_descriptor,
|
||||
i.conversation_agent_id, i.monitor_next_check_at, i.monitor_wake_requested_at, i.monitor_attempt_count,
|
||||
@@ -38,6 +40,7 @@ export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: st
|
||||
(i.assignee_agent_id IS NULL OR a.id IS NOT NULL) AS scoped,
|
||||
(${sql.raw(responseDrivenIssueWait('i'))}) AS response_driven_wait,
|
||||
(${sql.raw(completedAgentUnblockRequest('i'))}) AS unblock_request_finished,
|
||||
(${sql.raw(passiveIdleConversation('i'))}) AS passive_conversation,
|
||||
(SELECT jsonb_agg(jsonb_build_array(w.id, w.updated_at, w.kind, w.status, w.source_run_id,
|
||||
w.effective_resolver_policy, w.continuation_policy, w.addressee_agent_id, w.addressee_user_id) ORDER BY w.id)
|
||||
FROM issue_thread_interactions w WHERE w.issue_id = i.id AND w.status = 'pending') AS response_wait_generation,
|
||||
@@ -61,7 +64,7 @@ export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: st
|
||||
if (!statuses.has(row.status) || row.company_status !== "active" || row.scoped !== true
|
||||
|| (row.assignee_agent_id && !agentStatuses.has(row.agent_status ?? ""))
|
||||
|| row.unsafe_dependencies !== false || row.checkout_run_id || row.execution_run_id
|
||||
|| row.conversation_agent_id || row.monitor_wake_requested_at
|
||||
|| (row.conversation_agent_id && row.passive_conversation !== true) || row.monitor_wake_requested_at
|
||||
|| (row.execution_state !== null && state?.status !== "idle")
|
||||
|| (row.execution_policy !== null && !policy)) return { blocked: true };
|
||||
if (row.status === "blocked" && row.dependency_ready) return { blocked: true };
|
||||
@@ -81,7 +84,7 @@ export async function readIdleIssueSchedule(tx: Pick<Db, "execute">, ownerId: st
|
||||
}
|
||||
if (policy?.monitor != null) return { blocked: true };
|
||||
const unblock = record(row.unblock_descriptor);
|
||||
const passive = row.status === "backlog"
|
||||
const passive = row.passive_conversation === true || row.status === "backlog"
|
||||
|| (row.status === "todo" && !row.assignee_agent_id)
|
||||
|| (row.status === "todo" && row.assignee_agent_id && row.agent_status === "idle" && row.untouched_onboarding === true)
|
||||
|| (row.status === "blocked" && (row.unblock_descriptor === null || unblock?.owner === "board"
|
||||
|
||||
@@ -9,6 +9,23 @@ import { verifiedResourceCleanup } from "./cleanup-verification-proof.js";
|
||||
import { companyHoldMismatch } from "../modules/agent-lifecycle/index.js";
|
||||
import { pendingBudgetRunCancellationPredicate } from "./heartbeat/budget-cancellation.js";
|
||||
import { nativeResponsePresentationPendingQuery } from "./native-runtime/native-response-presentation-query.js";
|
||||
import { passiveIdleConversation } from "./conversation-idle-proof.js";
|
||||
import { idleChatHistoryChecks } from "./chat-channels/idle-history-proof.js";
|
||||
import { idleToolActionHistoryChecks } from "./tool-action-idle-proof.js";
|
||||
import { idleBrowserHistoryChecks } from "./browser-use-idle-proof.js";
|
||||
import { idleDecisionHistoryChecks } from "./decision-idle-proof.js";
|
||||
import { idleWorkspaceHistoryChecks } from "./workspace-history-idle-proof.js";
|
||||
import { idleMcpHistoryChecks } from "./public-mcp/idle-history-proof.js";
|
||||
import { idleExternalAgentHistoryChecks } from "../modules/external-agents/idle-history-proof.js";
|
||||
|
||||
export const completedHistoryChecks = [
|
||||
...idleChatHistoryChecks, ...idleToolActionHistoryChecks, ...idleBrowserHistoryChecks,
|
||||
...idleDecisionHistoryChecks, ...idleWorkspaceHistoryChecks, ...idleMcpHistoryChecks,
|
||||
...idleExternalAgentHistoryChecks,
|
||||
] as const;
|
||||
|
||||
const ordinaryIssues = `SELECT 1 FROM issues i WHERE i.status NOT IN ('done', 'cancelled')
|
||||
AND (${passiveIdleConversation('i')}) IS NOT TRUE`;
|
||||
|
||||
export interface IdleSleepDrainStatus {
|
||||
ownerId?: string;
|
||||
@@ -74,7 +91,7 @@ const WORK_CHECKS = [
|
||||
SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_id
|
||||
AND r.company_id = w.company_id AND r.agent_id = w.agent_id
|
||||
AND ${finishedRun("r")})))`,
|
||||
`SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`,
|
||||
ordinaryIssues,
|
||||
// A completed issue can still need its first watchdog review, including a
|
||||
// retry after immediate evaluation failed before creating a review/run.
|
||||
`SELECT 1 FROM issue_watchdogs WHERE status <> 'disabled'`,
|
||||
@@ -259,23 +276,7 @@ const WORK_CHECKS = [
|
||||
OR effect.delivery_state <> 'delivered' OR effect.delivered_at IS NULL
|
||||
OR effect.next_attempt_at IS NOT NULL))
|
||||
) IS NOT TRUE`,
|
||||
// These less common work sources fail closed on any retained state. Their
|
||||
// terminal-state exceptions can be added with tests for the owning service.
|
||||
...[
|
||||
"chat_deliveries", "chat_publications", "chat_actions",
|
||||
"chat_completion_deliveries", "tool_action_requests", "tool_action_deliveries",
|
||||
"workspace_runtime_services",
|
||||
"execution_workspace_runtime_leases",
|
||||
"company_transfer_runs", "decisions", "decision_effect_executions",
|
||||
"decision_archive_notification_outbox", "browser_use_sessions", "browser_use_runs",
|
||||
"browser_use_browsers",
|
||||
"company_secret_proposals",
|
||||
"mcp_oauth_grants", "mcp_mutation_receipts", "mcp_event_subscriptions",
|
||||
"mcp_event_deliveries", "mcp_attachment_uploads", "dot_agent_bindings",
|
||||
"dot_runner_assignments", "dot_runner_operations", "dot_mailbox_items",
|
||||
"muse_agent_bindings", "muse_credentials", "muse_runner_assignments",
|
||||
"muse_runner_operations", "muse_mailbox_items", "muse_input_deliveries", "external_agent_holds",
|
||||
].map((table) => `SELECT 1 FROM ${table}`),
|
||||
...completedHistoryChecks.map(check => check.query),
|
||||
] as const;
|
||||
|
||||
function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus, now: number): boolean {
|
||||
@@ -320,7 +321,7 @@ export async function readIdleSleepSafety(
|
||||
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
|
||||
const checks = WORK_CHECKS.filter(query =>
|
||||
(!scheduledRoutines || query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`)
|
||||
&& (!scheduledIssueMonitors || query !== `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`))
|
||||
&& (!scheduledIssueMonitors || query !== ordinaryIssues))
|
||||
.map((query) => sql`EXISTS (${sql.raw(query)})`);
|
||||
checks.push(sql`EXISTS (SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id
|
||||
WHERE ${companyHoldMismatch()})`);
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
/** Completion receipts are retained for idempotency. Fixed aliases only. */
|
||||
export const idleMcpHistoryChecks = [
|
||||
{ reason: "mcp_oauth_grants", query: `SELECT 1 FROM mcp_oauth_grants g WHERE
|
||||
(g.revoked_at IS NOT NULL AND EXISTS (SELECT 1 FROM companies c WHERE c.id = g.company_id)) IS NOT TRUE` },
|
||||
{ reason: "mcp_mutation_receipts", query: `SELECT 1 FROM mcp_mutation_receipts m WHERE (
|
||||
m.status = 'completed' AND jsonb_typeof(m.result) = 'object'
|
||||
AND EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = m.grant_id
|
||||
AND g.company_id = m.company_id AND g.user_id = m.user_id)) IS NOT TRUE` },
|
||||
// Stopped subscriptions no longer scan for events. Expiry alone is not a
|
||||
// handoff of responsibility for active subscriptions or unfinished delivery.
|
||||
{ reason: "mcp_event_subscriptions", query: `SELECT 1 FROM mcp_event_subscriptions s WHERE (
|
||||
s.stopped_at IS NOT NULL AND EXISTS (SELECT 1 FROM mcp_oauth_grants g
|
||||
WHERE g.id = s.grant_id AND g.company_id = s.company_id)) IS NOT TRUE` },
|
||||
{ reason: "mcp_event_deliveries", query: `SELECT 1 FROM mcp_event_deliveries d WHERE (
|
||||
d.finished_at IS NOT NULL AND (d.outcome IN ('delivered', 'filtered', 'inactive', 'delivery_exhausted')
|
||||
OR (d.outcome ~ '^http_[1-5][0-9][0-9]$' AND d.outcome !~ '^http_2'
|
||||
AND (d.attempts >= 6 OR (d.outcome NOT IN ('http_408', 'http_429') AND d.outcome !~ '^http_5'))))
|
||||
AND EXISTS (SELECT 1 FROM mcp_event_subscriptions s WHERE s.id = d.subscription_id
|
||||
AND EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = s.grant_id AND g.company_id = s.company_id)
|
||||
AND ((d.activity_id IS NOT NULL AND d.mailbox_item_id IS NULL AND EXISTS (
|
||||
SELECT 1 FROM activity_log a WHERE a.id = d.activity_id AND a.company_id = s.company_id))
|
||||
OR (d.mailbox_item_id IS NOT NULL AND d.activity_id IS NULL AND EXISTS (
|
||||
SELECT 1 FROM dot_mailbox_items m WHERE m.id = d.mailbox_item_id
|
||||
AND m.company_id = s.company_id AND m.binding_id = s.binding_id))))) IS NOT TRUE` },
|
||||
// The upload owner allocates object identity before storage writes and only
|
||||
// stamps cleaned_at after deletion. A committed attachment is never deleted.
|
||||
{ reason: "mcp_attachment_uploads", query: `SELECT 1 FROM mcp_attachment_uploads u WHERE (
|
||||
EXISTS (SELECT 1 FROM issues i WHERE i.id = u.task_id AND i.company_id = u.company_id)
|
||||
AND (u.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = u.grant_id
|
||||
AND g.company_id = u.company_id AND g.user_id = u.user_id))
|
||||
AND ((u.attachment_id IS NULL AND u.cleaned_at IS NOT NULL)
|
||||
OR (u.cleaned_at IS NULL AND u.attachment_id IS NOT NULL AND EXISTS (
|
||||
SELECT 1 FROM issue_attachments a JOIN assets stored ON stored.id = a.asset_id
|
||||
WHERE a.id = u.attachment_id AND a.company_id = u.company_id AND a.issue_id = u.task_id
|
||||
AND stored.company_id = u.company_id AND stored.object_key = u.object_key
|
||||
AND stored.provider = u.storage_provider)))) IS NOT TRUE` },
|
||||
] as const;
|
||||
@@ -0,0 +1,28 @@
|
||||
import { finishedIdleRun } from "./idle-work-proof.js";
|
||||
|
||||
function completedRequest(a: string): string {
|
||||
return `${a}.status IN ('executed', 'failed', 'rejected', 'expired', 'cancelled') AND ${a}.resolved_at IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM tool_invocations v WHERE v.id = ${a}.invocation_id AND v.company_id = ${a}.company_id
|
||||
AND v.issue_id IS NOT DISTINCT FROM ${a}.issue_id AND v.completed_at IS NOT NULL
|
||||
AND v.status IN ('succeeded', 'failed', 'denied', 'cancelled', 'timed_out', 'rate_limited')
|
||||
AND (v.run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = v.run_id
|
||||
AND r.company_id = v.company_id AND ${finishedIdleRun('r')})))
|
||||
AND (${a}.interaction_id IS NULL OR EXISTS (SELECT 1 FROM issue_thread_interactions i WHERE i.id = ${a}.interaction_id
|
||||
AND i.company_id = ${a}.company_id AND i.issue_id = ${a}.issue_id AND i.resolved_at IS NOT NULL
|
||||
AND i.payload->'toolAction'->>'actionRequestId' = ${a}.id::text
|
||||
AND i.payload->'toolAction'->>'invocationId' = ${a}.invocation_id::text
|
||||
AND i.status IN ('accepted', 'rejected', 'expired', 'cancelled')
|
||||
AND (${a}.status = 'rejected' OR i.result->'toolAction'->>'status' = CASE
|
||||
WHEN ${a}.status = 'cancelled' THEN 'expired' ELSE ${a}.status END)))`;
|
||||
}
|
||||
|
||||
export const idleToolActionHistoryChecks = [
|
||||
{ reason: "tool_action_requests", query: `SELECT 1 FROM tool_action_requests a WHERE (${completedRequest('a')}) IS NOT TRUE` },
|
||||
// delivered_at is the domain's acknowledgment. It also retires outcomes
|
||||
// after issue completion/reassignment. Accepted continuations stay covered by
|
||||
// the global wake/run checks, including coalesced run accounting.
|
||||
{ reason: "tool_action_deliveries", query: `SELECT 1 FROM tool_action_deliveries d WHERE (
|
||||
d.delivered_at IS NOT NULL AND EXISTS (SELECT 1 FROM tool_action_requests a
|
||||
WHERE a.id = d.action_request_id AND a.company_id = d.company_id
|
||||
AND a.issue_id = d.issue_id AND a.interaction_id = d.interaction_id AND (${completedRequest('a')}))) IS NOT TRUE` },
|
||||
] as const;
|
||||
@@ -0,0 +1,37 @@
|
||||
import { finishedIdleRun } from "./idle-work-proof.js";
|
||||
|
||||
export const idleWorkspaceHistoryChecks = [
|
||||
// Failed imports are intentionally resumable. Completed/cancelled rows can
|
||||
// still owe spool deletion, independently inspected by idle-local-work.
|
||||
{ reason: "company_transfer_runs", query: `SELECT 1 FROM company_transfer_runs t WHERE (
|
||||
t.status IN ('completed', 'cancelled') AND t.finished_at IS NOT NULL
|
||||
AND t.direction IN ('import', 'export') AND jsonb_typeof(t.container_ref) = 'object'
|
||||
AND jsonb_typeof(t.completed_parts) = 'array'
|
||||
AND (t.company_id IS NOT NULL OR (t.direction = 'import' AND t.status = 'cancelled'))
|
||||
AND (t.company_id IS NULL OR EXISTS (SELECT 1 FROM companies c WHERE c.id = t.company_id))) IS NOT TRUE` },
|
||||
{ reason: "workspace_runtime_services", query: `SELECT 1 FROM workspace_runtime_services s WHERE (
|
||||
s.status = 'stopped' AND s.stopped_at IS NOT NULL AND s.provider = 'local_process'
|
||||
AND s.exposure_handle IS NULL
|
||||
AND (s.exposure IS NULL OR (jsonb_typeof(s.exposure) = 'object'
|
||||
AND s.exposure->>'state' = 'removed' AND s.url IS NULL
|
||||
AND COALESCE(s.exposure->'publicUrl', 'null'::jsonb) = 'null'::jsonb))
|
||||
AND (s.execution_workspace_id IS NULL OR EXISTS (SELECT 1 FROM execution_workspaces w
|
||||
WHERE w.id = s.execution_workspace_id AND w.company_id = s.company_id))
|
||||
AND (s.issue_id IS NULL OR EXISTS (SELECT 1 FROM issues i WHERE i.id = s.issue_id AND i.company_id = s.company_id))
|
||||
AND (s.started_by_run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r
|
||||
WHERE r.id = s.started_by_run_id AND r.company_id = s.company_id AND ${finishedIdleRun('r')}))) IS NOT TRUE` },
|
||||
// This is an exclusivity row, not a provider lease. The domain only
|
||||
// evaluates staleness on the next claim; no expiry job is owed by history.
|
||||
{ reason: "execution_workspace_runtime_leases", query: `SELECT 1 FROM execution_workspace_runtime_leases l WHERE (
|
||||
EXISTS (SELECT 1 FROM execution_workspaces w WHERE w.id = l.execution_workspace_id AND w.company_id = l.company_id)
|
||||
AND ((l.owner_issue_id IS NOT NULL AND l.owner_key = 'issue:' || l.owner_issue_id::text
|
||||
AND EXISTS (SELECT 1 FROM issues i WHERE i.id = l.owner_issue_id AND i.company_id = l.company_id
|
||||
AND i.status IN ('done', 'cancelled')))
|
||||
OR (l.owner_issue_id IS NULL AND l.owner_run_id IS NOT NULL AND l.owner_key = 'run:' || l.owner_run_id::text
|
||||
AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = l.owner_run_id AND r.company_id = l.company_id
|
||||
AND r.agent_id = l.owner_agent_id AND ${finishedIdleRun('r')})))
|
||||
AND (l.owner_run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = l.owner_run_id
|
||||
AND r.company_id = l.company_id AND r.agent_id = l.owner_agent_id AND ${finishedIdleRun('r')}))
|
||||
AND NOT EXISTS (SELECT 1 FROM workspace_runtime_services s WHERE s.execution_workspace_id = l.execution_workspace_id
|
||||
AND (s.company_id <> l.company_id OR s.status <> 'stopped' OR s.stopped_at IS NULL))) IS NOT TRUE` },
|
||||
] as const;
|
||||
@@ -46,6 +46,7 @@ import {
|
||||
import { and, desc, eq, gte, inArray, isNull, lte, ne, or, sql } from "drizzle-orm";
|
||||
import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js";
|
||||
import { conflict, unprocessable } from "../errors.js";
|
||||
import { registerIdleLocalWorkProbe, type IdleLocalWork } from "./idle-local-work.js";
|
||||
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
|
||||
import { resolveHomeAwarePath } from "../home-paths.js";
|
||||
import { hasVerifiedWorktreeSeedManifest, isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
|
||||
@@ -283,6 +284,22 @@ const quarantinedRuntimeExposurePorts = new Set<number>();
|
||||
* companion together or not at all.
|
||||
*/
|
||||
const exposurePortPairClaims = new ExposurePortPairClaims();
|
||||
let runtimeReconstructionComplete = false;
|
||||
let runtimeReconstructionEpoch = 0;
|
||||
let runtimeReconciliation: ReturnType<typeof reconstructRuntimeServices> | null = null;
|
||||
|
||||
/** Read owner memory only. Durable stopped rows do not describe a surviving
|
||||
* registry process, an allocation being published, or unfinished exposure cleanup. */
|
||||
export function readWorkspaceRuntimeIdleWork(): IdleLocalWork {
|
||||
if (!runtimeReconstructionComplete || runtimeReconciliation !== null) return "unknown";
|
||||
return runtimeServicesById.size || runtimeProvisionByWorkspace.size || runtimeControlStartByOwner.size
|
||||
|| runtimeReplacementClaimsByReuseKey.size || quarantinedRuntimeExposurePorts.size
|
||||
|| exposurePortPairClaims.activePorts().size
|
||||
|| [...inFlightAllocatedPorts.values()].some(until => until > Date.now()) ? "present" : "none";
|
||||
}
|
||||
// This registry and its probe share the application process lifetime, like the
|
||||
// native-session owner. Stopping one resource must not unregister its siblings.
|
||||
registerIdleLocalWorkProbe(readWorkspaceRuntimeIdleWork);
|
||||
/**
|
||||
* Execution-workspace statuses that still hold an exclusive lease. `archived`
|
||||
* is the only terminal state; everything else — including `idle` — is a lane an
|
||||
@@ -525,6 +542,8 @@ type ProcessOutputAccumulator = {
|
||||
export async function resetRuntimeServicesForTests(
|
||||
opts: { terminateProcesses?: boolean; simulateSupervisorExit?: boolean } = {},
|
||||
) {
|
||||
runtimeReconstructionComplete = false;
|
||||
runtimeReconstructionEpoch += 1;
|
||||
if (opts.terminateProcesses) {
|
||||
for (const serviceId of [...runtimeServicesById.keys()]) {
|
||||
await stopRuntimeService(serviceId).catch(() => undefined);
|
||||
@@ -8577,6 +8596,19 @@ export async function refreshPersistedRuntimeServiceHealth(input: {
|
||||
}
|
||||
|
||||
export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
if (runtimeReconciliation) return runtimeReconciliation;
|
||||
runtimeReconstructionComplete = false;
|
||||
const epoch = ++runtimeReconstructionEpoch;
|
||||
runtimeReconciliation = reconstructRuntimeServices(db).then(result => {
|
||||
if (epoch === runtimeReconstructionEpoch) {
|
||||
runtimeReconstructionComplete = result.restartFailed === 0 && result.exposureReservationDrift.length === 0;
|
||||
}
|
||||
return result;
|
||||
}).finally(() => { runtimeReconciliation = null; });
|
||||
return runtimeReconciliation;
|
||||
}
|
||||
|
||||
async function reconstructRuntimeServices(db: Db) {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(workspaceRuntimeServices)
|
||||
|
||||
Reference in new issue
Block a user