mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 02:32:07 +01:00
8d9d2f9ddf
Preserve deliberate assignee blocks and operator stops during dependency and run-release recovery. Attribute native waits to their bound owner. Ignore unrelated edits and shared-workspace exports from another issue. Continue after genuine dependency changes while preserving independently authorized user input. Keep the original contributor's implementation and review history. Add native ownership, unrelated-edit, later readiness, and shared-workspace regressions. The shared-workspace loop was reproduced before the fix. Co-Authored-By: Devin Foley <devin@paperclip.ing> Co-Authored-By: Paperclip <noreply@paperclip.ing>
6377 lines
226 KiB
TypeScript
6377 lines
226 KiB
TypeScript
import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js";
|
||
import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js";
|
||
import { isAiAuthenticationBlocked } from "../ai-auth-failure.js";
|
||
import { hasCommittedNativePlanWait } from "../native-runtime/native-plan-wait.js";
|
||
import { isNativeWorkspaceExportRepairCause } from "@paperclipai/shared";
|
||
import { settleSlackConversation } from "../slack-conversation-lifecycle.js";
|
||
import { externalConversationStateSql } from "../slack-conversation-state.js";
|
||
import { isIssueReviewPathRecoveryRun } from "./review-path-recovery.js";
|
||
import { escalateExhaustedIssueReviewPathRecovery } from "./review-path-recovery-escalation.js";
|
||
import { executionRetryAccounting } from "../execution-recovery-attempt.js";
|
||
import { isExplicitContinuationRetryClaim } from "../explicit-continuation-retry-claim.js";
|
||
import {
|
||
decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint,
|
||
LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode,
|
||
} from "./legacy-continuation.js";
|
||
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController } from "../legacy-controller-lease.js";
|
||
import { isCompletedOnboardingHandoffWake } from "../chat-completion-delivery.js";
|
||
import { instanceSettingsService } from "../instance-settings.js";
|
||
import { isWaitingConversation, settleConversationTurn, deliverConversationComments } from "../agent-conversations.js";
|
||
import {
|
||
and,
|
||
asc,
|
||
desc,
|
||
eq,
|
||
gt,
|
||
gte,
|
||
inArray,
|
||
isNotNull,
|
||
isNull,
|
||
not,
|
||
notExists,
|
||
notInArray,
|
||
or,
|
||
sql,
|
||
} from "drizzle-orm";
|
||
import type { Db } from "@paperclipai/db";
|
||
import {
|
||
hasCommittedNativeBoardResponseWait,
|
||
readNativeBoardResponseWaitSource,
|
||
} from "../native-runtime/native-board-response-wait.js";
|
||
import { authorizeChatConversationForBoundRun } from "../native-runtime/chat-attachment-reuse.js";
|
||
import {
|
||
ONBOARDING_FIRST_TASK_ORIGIN_KIND,
|
||
PROVIDER_QUOTA_MONITOR_SERVICE_NAME,
|
||
ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||
requiresExecutionReconciliation,
|
||
type IssueCommentMetadata,
|
||
type IssueCommentPresentation,
|
||
} from "@paperclipai/shared";
|
||
import {
|
||
agents,
|
||
agentTaskSessions,
|
||
agentWakeupRequests,
|
||
approvals,
|
||
activityLog,
|
||
chatConversations,
|
||
companies,
|
||
heartbeatRunWatchdogDecisions,
|
||
heartbeatRuns,
|
||
issueAttachments,
|
||
issueComments,
|
||
issueApprovals,
|
||
issueRecoveryActions,
|
||
issueRelations,
|
||
issueThreadInteractions,
|
||
issues,
|
||
nativeRunFinalizations,
|
||
nativeRunResults,
|
||
statusDecisions,
|
||
routines,
|
||
workAssessments,
|
||
workspaceOperations,
|
||
} from "@paperclipai/db";
|
||
import { parseObject, asBoolean, asNumber } from "../../adapters/utils.js";
|
||
import { runningProcesses } from "../../adapters/index.js";
|
||
import {
|
||
isNativeRunnerOwnershipHeld,
|
||
nativeRunnerOwnershipNotHeldCondition,
|
||
} from "../native-runtime/native-runner-ownership.js";
|
||
import { visibleIssueCondition } from "../issue-visibility.js";
|
||
import { forbidden, notFound } from "../../errors.js";
|
||
import { logger } from "../../middleware/logger.js";
|
||
import {
|
||
isPidAlive,
|
||
isProcessGroupAlive,
|
||
} from "../local-service-supervisor.js";
|
||
import { redactSensitiveText } from "../../redaction.js";
|
||
import { isUniqueViolation } from "../../db-errors.js";
|
||
import {
|
||
logActivity,
|
||
publishActivity,
|
||
type ActivityPublication,
|
||
} from "../activity-log.js";
|
||
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||
import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";
|
||
import { budgetService } from "../budgets.js";
|
||
import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js";
|
||
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||
import { hasUnverifiedNativeWorkspaceFinalizationOwner } from "../native-runtime/native-workspace-finalization-recovery-state.js";
|
||
import {
|
||
legacyExecutionNeedsReconciliationWithEvidence,
|
||
terminalizeLegacyExecution,
|
||
} from "../legacy-execution-recovery.js";
|
||
import { issueTreeControlService } from "../issue-tree-control.js";
|
||
import { isExternalChatPresentationContext } from "../heartbeat-run-summary.js";
|
||
import {
|
||
CHAT_CONTROL_RECOVERY_STOP_CODE,
|
||
CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,
|
||
readChatControlRecoveryStop,
|
||
} from "../chat-control-recovery-stop.js";
|
||
import {
|
||
TERMINAL_HEARTBEAT_RUN_STATUSES,
|
||
issueService,
|
||
executeIssuePostCommitActions,
|
||
type IssuePostCommitAction,
|
||
} from "../issues.js";
|
||
import {
|
||
applyIssueMonitorPolicyTransition,
|
||
normalizeIssueExecutionPolicy,
|
||
parseIssueExecutionState,
|
||
} from "../issue-execution-policy.js";
|
||
import {
|
||
ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
|
||
buildIssueBlockersResolvedWakeStateKey,
|
||
findExistingIssueBlockersResolvedWakeForReadyState,
|
||
} from "../issue-dependency-wakeups.js";
|
||
import { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js";
|
||
import { isHeartbeatWakeOnDemandEnabled } from "../heartbeat-policy.js";
|
||
import {
|
||
DEFAULT_MAX_SUCCESSFUL_RUN_HANDOFF_ATTEMPTS,
|
||
FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,
|
||
SUCCESSFUL_RUN_MISSING_STATE_REASON,
|
||
buildSuccessfulRunHandoffExhaustedNotice,
|
||
isPluginManagedIssueLifecycle,
|
||
noticeMetadataReferencesRecoveryAction,
|
||
type SuccessfulRunHandoffNotice,
|
||
} from "./successful-run-handoff.js";
|
||
import {
|
||
SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON,
|
||
sandboxProviderPluginRemedy,
|
||
buildExecutionReviewParticipantRecoveryNoticeSeed,
|
||
buildExecutionReviewParticipantUnavailableNoticeSeed,
|
||
buildStrandedRecoveryEscalationNotice,
|
||
type StrandedRecoveryNoticeSeed,
|
||
} from "./stranded-notice.js";
|
||
import {
|
||
RECOVERY_ORIGIN_KINDS,
|
||
isStrandedIssueRecoveryOriginKind,
|
||
} from "./origins.js";
|
||
import { withRecoveryContext } from "./status-only-context.js";
|
||
import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js";
|
||
import {
|
||
collectDispositionRepairSourceState,
|
||
dispositionRepairDelayMs,
|
||
DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
} from "./disposition-repair.js";
|
||
import {
|
||
createActiveRunWatchdog,
|
||
WatchdogDecisionApplicationError,
|
||
type RunOutputSilenceSummary,
|
||
type WatchdogDecisionActor,
|
||
} from "../../modules/active-run-watchdog/index.js";
|
||
|
||
const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = [
|
||
"queued",
|
||
"running",
|
||
"scheduled_retry",
|
||
] as const;
|
||
const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = [
|
||
"interrupted",
|
||
"failed",
|
||
"cancelled",
|
||
"timed_out",
|
||
] as const;
|
||
export const ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS = 5 * 60 * 1000;
|
||
export const ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS = 15 * 60 * 1000;
|
||
export const ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS = 30 * 60 * 1000;
|
||
const STRANDED_ISSUE_RECOVERY_ORIGIN_KIND =
|
||
RECOVERY_ORIGIN_KINDS.strandedIssueRecovery;
|
||
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
|
||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON =
|
||
"execution_review_participant_recovery";
|
||
const STRANDED_BOARD_ESCALATION_POLICY = "board_escalation_no_takeover_v1";
|
||
const DISPOSITION_REPAIR_IDEMPOTENCY_INDEX =
|
||
"agent_wakeup_requests_disposition_repair_idempotency_uq";
|
||
const RESOLVED_DEPENDENCY_WAKE_BACKSTOP_CANDIDATE_LIMIT = 500;
|
||
|
||
// GGU-809: when a stranded `in_progress` issue would otherwise hit the
|
||
// `isRepeatedProductiveContinuationRecovery` escalation path, exempt the
|
||
// escalation if the assignee posted a comment or attachment within this window.
|
||
// Batch workflows (e.g. Image Spec multi-frame generation) make real progress
|
||
// every heartbeat and would otherwise trigger a recovery issue after just two
|
||
// productive heartbeats. Floor the override at 60s to keep the exemption from
|
||
// being effectively disabled by misconfiguration.
|
||
export const STRANDED_RECENT_PROGRESS_EXEMPTION_MS = Math.max(
|
||
60_000,
|
||
Number(process.env.STRANDED_RECENT_PROGRESS_EXEMPTION_MS) || 30 * 60 * 1000,
|
||
);
|
||
|
||
type RecoveryWakeupOptions = {
|
||
source?: "timer" | "assignment" | "on_demand" | "automation";
|
||
triggerDetail?: "manual" | "ping" | "callback" | "system";
|
||
reason?: string | null;
|
||
payload?: Record<string, unknown> | null;
|
||
idempotencyKey?: string | null;
|
||
requestedByActorType?: "user" | "agent" | "system";
|
||
requestedByActorId?: string | null;
|
||
contextSnapshot?: Record<string, unknown>;
|
||
issueStateGuard?: {
|
||
statuses: string[];
|
||
assigneeAgentId: string;
|
||
};
|
||
};
|
||
|
||
type RecoveryWakeup = (
|
||
agentId: string,
|
||
opts?: RecoveryWakeupOptions,
|
||
) => Promise<typeof heartbeatRuns.$inferSelect | null>;
|
||
|
||
type ResolvedDependencyWakeBackstopSource =
|
||
"issue_graph_liveness.backstop" | "workspace.finalize";
|
||
|
||
type ResolvedDependencyWakeBackstopOptions = {
|
||
runId?: string | null;
|
||
companyId?: string | null;
|
||
blockerIssueId?: string | null;
|
||
source?: ResolvedDependencyWakeBackstopSource;
|
||
};
|
||
|
||
type LatestIssueRun =
|
||
| (Pick<
|
||
typeof heartbeatRuns.$inferSelect,
|
||
| "id"
|
||
| "agentId"
|
||
| "status"
|
||
| "error"
|
||
| "errorCode"
|
||
| "contextSnapshot"
|
||
| "livenessState"
|
||
| "startedAt"
|
||
| "createdAt"
|
||
> & {
|
||
resultJson?: unknown;
|
||
})
|
||
| null;
|
||
type SuccessfulLatestIssueRun = NonNullable<LatestIssueRun> & {
|
||
status: "succeeded";
|
||
};
|
||
|
||
export type StrandedRecoveryCause =
|
||
| "stranded_assigned_issue"
|
||
| "deliberate_wait_without_target"
|
||
| "process_lost"
|
||
| "provider_quota"
|
||
| "codex_output_inactivity_monitor"
|
||
| "workspace_validation_failed"
|
||
| "configuration_incomplete"
|
||
| "native_session_interrupted"
|
||
| "native_runner_process_exited"
|
||
| "provider_transport_failed"
|
||
| "provider_frame_too_large"
|
||
| "execution_review_participant_recovery"
|
||
| typeof SUCCESSFUL_RUN_MISSING_STATE_REASON;
|
||
|
||
const NATIVE_RUNNER_RECOVERY_CAUSES = new Set<StrandedRecoveryCause>([
|
||
"native_session_interrupted",
|
||
"native_runner_process_exited",
|
||
"provider_transport_failed",
|
||
"provider_frame_too_large",
|
||
]);
|
||
|
||
export function shouldRouteRecoveryToOriginalAgent(
|
||
cause: StrandedRecoveryCause,
|
||
): boolean {
|
||
return (
|
||
cause === "process_lost" ||
|
||
cause === SUCCESSFUL_RUN_MISSING_STATE_REASON ||
|
||
cause === "codex_output_inactivity_monitor" ||
|
||
NATIVE_RUNNER_RECOVERY_CAUSES.has(cause)
|
||
);
|
||
}
|
||
|
||
type StrandedPreviousStatus = "todo" | "in_progress" | "in_review";
|
||
|
||
type SuccessfulRunHandoffRecoveryEvidence = {
|
||
sourceRunId: string | null;
|
||
correctiveRunId: string;
|
||
missingDisposition: string;
|
||
handoffAttempt: number;
|
||
maxHandoffAttempts: number;
|
||
};
|
||
|
||
function compactRecoveryPresentation(title: string): IssueCommentPresentation {
|
||
const normalizedTitle = title.trim();
|
||
return {
|
||
kind: "system_notice",
|
||
tone: "warning",
|
||
title:
|
||
normalizedTitle.length > 160
|
||
? `${normalizedTitle.slice(0, 159)}…`
|
||
: normalizedTitle,
|
||
detailsDefaultOpen: false,
|
||
density: "compact",
|
||
};
|
||
}
|
||
|
||
function recoveryCauseTitle(cause: StrandedRecoveryCause) {
|
||
switch (cause) {
|
||
case "process_lost":
|
||
return "retries exhausted";
|
||
case "codex_output_inactivity_monitor":
|
||
return "output-inactivity retry exhausted";
|
||
case "workspace_validation_failed":
|
||
return "workspace validation failed";
|
||
case "configuration_incomplete":
|
||
return "configuration incomplete";
|
||
case "execution_review_participant_recovery":
|
||
return "reviewer recovery failed";
|
||
case "provider_quota":
|
||
return "provider quota unavailable";
|
||
case SUCCESSFUL_RUN_MISSING_STATE_REASON:
|
||
return "missing disposition recovery failed";
|
||
default:
|
||
return "execution path recovery failed";
|
||
}
|
||
}
|
||
|
||
function recoveryNoticeMetadata(input: {
|
||
cause: string;
|
||
latestRun: LatestIssueRun;
|
||
recoveryActionId?: string | null;
|
||
previousStatus: string;
|
||
recoveryOwner?: Pick<typeof agents.$inferSelect, "id" | "name"> | null;
|
||
}): IssueCommentMetadata {
|
||
const rows: IssueCommentMetadata["sections"][number]["rows"] = [
|
||
...(input.recoveryActionId
|
||
? [
|
||
{
|
||
type: "key_value" as const,
|
||
label: "Recovery action",
|
||
value: input.recoveryActionId,
|
||
},
|
||
]
|
||
: []),
|
||
{ type: "key_value", label: "Cause", value: input.cause },
|
||
{
|
||
type: "key_value",
|
||
label: "Previous status",
|
||
value: input.previousStatus,
|
||
},
|
||
...(input.recoveryOwner
|
||
? [
|
||
{
|
||
type: "agent_link" as const,
|
||
label: "Recovery owner",
|
||
agentId: input.recoveryOwner.id,
|
||
name: input.recoveryOwner.name.slice(0, 160),
|
||
},
|
||
]
|
||
: [
|
||
{
|
||
type: "key_value" as const,
|
||
label: "Recovery owner",
|
||
value: "board",
|
||
},
|
||
]),
|
||
...(input.latestRun
|
||
? [
|
||
{
|
||
type: "run_link" as const,
|
||
label: "Latest run",
|
||
runId: input.latestRun.id,
|
||
title: input.latestRun.status,
|
||
},
|
||
]
|
||
: []),
|
||
];
|
||
|
||
return {
|
||
version: 1,
|
||
sourceRunId: input.latestRun?.id ?? null,
|
||
sections: [{ title: "Recovery", rows }],
|
||
};
|
||
}
|
||
|
||
function readRecoveryRunErrorFamily(latestRun: LatestIssueRun) {
|
||
const result = parseObject(latestRun?.resultJson);
|
||
return readNonEmptyString(result.errorFamily);
|
||
}
|
||
|
||
function isProviderQuotaRecovery(latestRun: LatestIssueRun) {
|
||
if (latestRun?.errorCode === "provider_quota") return true;
|
||
if (readRecoveryRunErrorFamily(latestRun) === "provider_quota") return true;
|
||
if (latestRun?.errorCode !== "adapter_failed") return false;
|
||
return /(?:usage|rate|quota) limit|you(?:'|’)ve hit your (?:\w+ )?limit|quota (?:exceeded|reset)|try again after/i.test(
|
||
latestRun.error ?? "",
|
||
);
|
||
}
|
||
|
||
function resolveStrandedRecoveryCause(
|
||
latestRun: LatestIssueRun,
|
||
explicitCause?: StrandedRecoveryCause,
|
||
): StrandedRecoveryCause {
|
||
if (explicitCause) return explicitCause;
|
||
if (isProviderQuotaRecovery(latestRun)) return "provider_quota";
|
||
if (latestRun?.errorCode === "process_lost") return "process_lost";
|
||
if (latestRun?.errorCode === "codex_output_inactivity_monitor") {
|
||
return "codex_output_inactivity_monitor";
|
||
}
|
||
if (
|
||
NATIVE_RUNNER_RECOVERY_CAUSES.has(
|
||
latestRun?.errorCode as StrandedRecoveryCause,
|
||
)
|
||
) {
|
||
return latestRun!.errorCode as StrandedRecoveryCause;
|
||
}
|
||
return "stranded_assigned_issue";
|
||
}
|
||
|
||
function readWorkspaceValidationPayload(
|
||
latestRun: LatestIssueRun,
|
||
): Record<string, unknown> | null {
|
||
const payload = parseObject(
|
||
parseObject(latestRun?.resultJson).workspaceValidation,
|
||
);
|
||
return Object.keys(payload).length > 0 ? payload : null;
|
||
}
|
||
|
||
function readWorkspaceValidationFingerprint(
|
||
latestRun: LatestIssueRun,
|
||
): string | null {
|
||
const payload = readWorkspaceValidationPayload(latestRun);
|
||
return readNonEmptyString(payload?.fingerprint);
|
||
}
|
||
|
||
function readConfigurationIncompletePayload(
|
||
latestRun: LatestIssueRun,
|
||
): Record<string, unknown> | null {
|
||
const payload = parseObject(
|
||
parseObject(latestRun?.resultJson).configurationIncomplete,
|
||
);
|
||
return Object.keys(payload).length > 0 ? payload : null;
|
||
}
|
||
|
||
function readConfigurationIncompleteFingerprint(
|
||
latestRun: LatestIssueRun,
|
||
): string | null {
|
||
return readNonEmptyString(
|
||
readConfigurationIncompletePayload(latestRun)?.fingerprint,
|
||
);
|
||
}
|
||
|
||
export type { RunOutputSilenceSummary, WatchdogDecisionActor };
|
||
|
||
function readNonEmptyString(value: unknown): string | null {
|
||
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
||
}
|
||
|
||
function summarizeRunFailureForIssueComment(run: LatestIssueRun) {
|
||
if (!run) return null;
|
||
|
||
if (readNonEmptyString(run.error) || readNonEmptyString(run.errorCode)) {
|
||
return " Latest retry failure details were withheld from the issue thread; inspect the linked run for evidence.";
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function didAutomaticRecoveryFail(
|
||
latestRun: LatestIssueRun,
|
||
expectedRetryReason:
|
||
| "assignment_recovery"
|
||
| "issue_continuation_needed"
|
||
| typeof EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
|
||
) {
|
||
if (!latestRun) return false;
|
||
|
||
const latestContext = parseObject(latestRun.contextSnapshot);
|
||
const latestRetryReason = readNonEmptyString(latestContext.retryReason);
|
||
return (
|
||
latestRetryReason === expectedRetryReason &&
|
||
UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.includes(
|
||
latestRun.status as (typeof UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES)[number],
|
||
)
|
||
);
|
||
}
|
||
|
||
function isTerminalIssueRun(latestRun: LatestIssueRun) {
|
||
if (!latestRun) return false;
|
||
return TERMINAL_HEARTBEAT_RUN_STATUSES.has(latestRun.status);
|
||
}
|
||
|
||
const TRANSIENT_INFRA_CONTINUATION_ERROR_CODES = new Set<string>([
|
||
"adapter_failed",
|
||
"codex_transient_upstream",
|
||
"codex_harness_crash",
|
||
"claude_transient_upstream",
|
||
"provider_quota",
|
||
"timeout",
|
||
// The host lost run ownership before dispatch; no provider work began.
|
||
CONTROLLER_LEASE_LOST_FAILURE_CODE,
|
||
]);
|
||
|
||
const NON_RETRYABLE_CONTINUATION_ERROR_CODES = new Set<string>([
|
||
"native_provider_model_rejected",
|
||
"provider_tool_definition_invalid",
|
||
"adapter_engine_unavailable",
|
||
"agent_not_invokable",
|
||
"agent_not_found",
|
||
"budget_blocked",
|
||
"budget_exhausted",
|
||
"issue_paused",
|
||
"issue_dependencies_blocked",
|
||
// This is the fail-closed fallback for exceptions raised before an adapter
|
||
// process starts. Known transient preflight failures use dedicated bounded
|
||
// retry paths instead of generic issue continuation recovery.
|
||
"setup_failed",
|
||
// Setup owns the shared durable retry budget for temporary Git scans.
|
||
// Generic continuation must not retry permanent failures or reset that budget.
|
||
"workspace_git_scan_timeout",
|
||
"workspace_git_scan_saturated",
|
||
"workspace_git_scan_cancelled",
|
||
"workspace_git_scan_output_limit",
|
||
"workspace_git_scan_failed",
|
||
"low_trust_isolation_unavailable",
|
||
"low_trust_requires_isolated_workspace",
|
||
"low_trust_boundary_mismatch",
|
||
"low_trust_requires_sandbox_environment",
|
||
"low_trust_runtime_services_denied",
|
||
]);
|
||
|
||
// A continuation cancelled with this code is a *deliberate wait* (the latest run
|
||
// reported it was parked for review/approval), not a lost execution path. When the
|
||
// issue has a real waiting target we convert it into a normal dependency wait rather
|
||
// than escalating it as stranded.
|
||
const CONTINUATION_WAITING_ON_REVIEW_ERROR_CODE =
|
||
"issue_continuation_waiting_on_review";
|
||
const INTERACTION_CONTINUATION_REQUEUE_MAX_ATTEMPTS = 3;
|
||
|
||
const CONTINUATION_RECOVERY_TRANSIENT_MAX_ATTEMPTS = 3;
|
||
const CONTINUATION_RECOVERY_DEFAULT_MAX_ATTEMPTS = 1;
|
||
const CONTINUATION_RECOVERY_TRANSIENT_BASE_BACKOFF_MS = 60_000;
|
||
export const PROVIDER_QUOTA_RECOVERY_DEFAULT_BACKOFF_MS = 60 * 60 * 1000;
|
||
|
||
const PROVIDER_QUOTA_ERROR_RE =
|
||
/(?:you(?:'|’)ve hit your (?:\w+ )?limit|usage limit(?: reached| exceeded)?|provider quota|quota (?:limit )?exceeded|model (?:is )?at capacity)/i;
|
||
const CONFIGURATION_INCOMPLETE_ERROR_RE =
|
||
/(?:model_not_found|model [^\n]{0,120} not found|missing (?:api )?(?:key|credentials?)|credentials? (?:are |is )?missing|no (?:api )?(?:key|credentials?) (?:was |were )?(?:found|configured|provided)|api key (?:is )?(?:not set|unavailable))/i;
|
||
|
||
export type AdapterFailureRecoveryClassification =
|
||
| { kind: "provider_quota"; retryAt: Date; parsedResetTime: boolean }
|
||
| { kind: "configuration_incomplete" }
|
||
| null;
|
||
|
||
function parseProviderQuotaClockReset(error: string, now: Date) {
|
||
const match = error.match(
|
||
/(?:try again at|resets?(?:\s+at)?)\s+(\d{1,2})(?::(\d{2}))?\s*(?:([ap])\.?\s*m\.?)?(?:\s*\(([^)]+)\)|\s+([A-Z]{2,5}))?/i,
|
||
);
|
||
if (!match) return null;
|
||
|
||
const hourValue = Number.parseInt(match[1] ?? "", 10);
|
||
const minute = Number.parseInt(match[2] ?? "0", 10);
|
||
const meridiem = (match[3] ?? "").toLowerCase();
|
||
if (!Number.isInteger(hourValue)) return null;
|
||
if (
|
||
meridiem ? hourValue < 1 || hourValue > 12 : hourValue < 0 || hourValue > 23
|
||
)
|
||
return null;
|
||
if (!Number.isInteger(minute) || minute < 0 || minute > 59) return null;
|
||
|
||
let hour = meridiem ? hourValue % 12 : hourValue;
|
||
if (meridiem === "p") hour += 12;
|
||
const timeZone = (match[4] ?? match[5])?.trim();
|
||
if (!timeZone) {
|
||
const retryAt = new Date(now);
|
||
retryAt.setUTCHours(hour, minute, 0, 0);
|
||
if (retryAt.getTime() <= now.getTime())
|
||
retryAt.setUTCDate(retryAt.getUTCDate() + 1);
|
||
return retryAt;
|
||
}
|
||
|
||
try {
|
||
const wallClock = (date: Date) =>
|
||
Object.fromEntries(
|
||
new Intl.DateTimeFormat("en-US", {
|
||
timeZone,
|
||
hourCycle: "h23",
|
||
year: "numeric",
|
||
month: "2-digit",
|
||
day: "2-digit",
|
||
hour: "2-digit",
|
||
minute: "2-digit",
|
||
})
|
||
.formatToParts(date)
|
||
.map((part) => [part.type, part.value]),
|
||
);
|
||
const nowParts = wallClock(now);
|
||
const buildRetryAt = (dayOffset: number) => {
|
||
const targetDay = new Date(
|
||
Date.UTC(
|
||
Number(nowParts.year),
|
||
Number(nowParts.month) - 1,
|
||
Number(nowParts.day) + dayOffset,
|
||
hour,
|
||
minute,
|
||
),
|
||
);
|
||
let candidate = targetDay;
|
||
const targetMs = targetDay.getTime();
|
||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||
const actual = wallClock(candidate);
|
||
const actualMs = Date.UTC(
|
||
Number(actual.year),
|
||
Number(actual.month) - 1,
|
||
Number(actual.day),
|
||
Number(actual.hour),
|
||
Number(actual.minute),
|
||
);
|
||
const adjustment = targetMs - actualMs;
|
||
if (adjustment === 0) break;
|
||
candidate = new Date(candidate.getTime() + adjustment);
|
||
}
|
||
return candidate;
|
||
};
|
||
const sameDay = buildRetryAt(0);
|
||
return sameDay.getTime() > now.getTime() ? sameDay : buildRetryAt(1);
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
export function classifyAdapterFailureForRecovery(
|
||
latestRun: Pick<
|
||
NonNullable<LatestIssueRun>,
|
||
"error" | "errorCode" | "resultJson"
|
||
>,
|
||
now = new Date(),
|
||
): AdapterFailureRecoveryClassification {
|
||
// An engine prerequisite cannot be repaired by asking the same unavailable
|
||
// engine to retry. Use the existing configuration-blocker path.
|
||
if (latestRun.errorCode === "adapter_engine_unavailable" || latestRun.errorCode === "provider_tool_definition_invalid" || latestRun.errorCode === "native_provider_model_rejected") {
|
||
return { kind: "configuration_incomplete" };
|
||
}
|
||
if (
|
||
latestRun.errorCode !== "adapter_failed" &&
|
||
latestRun.errorCode !== "provider_quota" &&
|
||
latestRun.errorCode !== "configuration_incomplete"
|
||
) {
|
||
return null;
|
||
}
|
||
const resultJson = parseObject(latestRun.resultJson);
|
||
const error = [
|
||
latestRun.errorCode ?? "",
|
||
latestRun.error ?? "",
|
||
JSON.stringify(resultJson),
|
||
].join("\n");
|
||
if (
|
||
latestRun.errorCode === "configuration_incomplete" ||
|
||
CONFIGURATION_INCOMPLETE_ERROR_RE.test(error)
|
||
) {
|
||
return { kind: "configuration_incomplete" };
|
||
}
|
||
if (
|
||
latestRun.errorCode !== "provider_quota" &&
|
||
!PROVIDER_QUOTA_ERROR_RE.test(error)
|
||
)
|
||
return null;
|
||
|
||
const persistedRetryAt =
|
||
readNonEmptyString(resultJson.retryNotBefore) ??
|
||
readNonEmptyString(resultJson.transientRetryNotBefore) ??
|
||
readNonEmptyString(resultJson.providerQuotaRetryNotBefore);
|
||
const parsedPersistedRetryAt = persistedRetryAt
|
||
? new Date(persistedRetryAt)
|
||
: null;
|
||
if (
|
||
parsedPersistedRetryAt &&
|
||
!Number.isNaN(parsedPersistedRetryAt.getTime()) &&
|
||
parsedPersistedRetryAt > now
|
||
) {
|
||
return {
|
||
kind: "provider_quota",
|
||
retryAt: parsedPersistedRetryAt,
|
||
parsedResetTime: true,
|
||
};
|
||
}
|
||
|
||
const parsedClockReset = parseProviderQuotaClockReset(error, now);
|
||
if (parsedClockReset) {
|
||
return {
|
||
kind: "provider_quota",
|
||
retryAt: parsedClockReset,
|
||
parsedResetTime: true,
|
||
};
|
||
}
|
||
return {
|
||
kind: "provider_quota",
|
||
retryAt: new Date(
|
||
now.getTime() + PROVIDER_QUOTA_RECOVERY_DEFAULT_BACKOFF_MS,
|
||
),
|
||
parsedResetTime: false,
|
||
};
|
||
}
|
||
|
||
type ContinuationRetryClassification = {
|
||
kind:
|
||
| "transient_infra"
|
||
| "non_retryable"
|
||
| "deliberate_wait_without_target"
|
||
| "default";
|
||
maxAttempts: number;
|
||
baseBackoffMs: number;
|
||
errorCode: string | null;
|
||
};
|
||
|
||
export function classifyContinuationFailure(
|
||
latestRun: LatestIssueRun,
|
||
): ContinuationRetryClassification {
|
||
const errorCode = readNonEmptyString(latestRun?.errorCode);
|
||
if (errorCode === CONTINUATION_WAITING_ON_REVIEW_ERROR_CODE) {
|
||
return {
|
||
kind: "deliberate_wait_without_target",
|
||
maxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
baseBackoffMs: CONTINUATION_RECOVERY_TRANSIENT_BASE_BACKOFF_MS,
|
||
errorCode,
|
||
};
|
||
}
|
||
if (isAiAuthenticationBlocked(latestRun) || (errorCode && NON_RETRYABLE_CONTINUATION_ERROR_CODES.has(errorCode))) {
|
||
return {
|
||
kind: "non_retryable",
|
||
maxAttempts: 0,
|
||
baseBackoffMs: 0,
|
||
errorCode,
|
||
};
|
||
}
|
||
if (errorCode && TRANSIENT_INFRA_CONTINUATION_ERROR_CODES.has(errorCode)) {
|
||
return {
|
||
kind: "transient_infra",
|
||
maxAttempts: CONTINUATION_RECOVERY_TRANSIENT_MAX_ATTEMPTS,
|
||
baseBackoffMs: CONTINUATION_RECOVERY_TRANSIENT_BASE_BACKOFF_MS,
|
||
errorCode,
|
||
};
|
||
}
|
||
return {
|
||
kind: "default",
|
||
maxAttempts: CONTINUATION_RECOVERY_DEFAULT_MAX_ATTEMPTS,
|
||
baseBackoffMs: 0,
|
||
errorCode,
|
||
};
|
||
}
|
||
|
||
function successfulRunHandoffRecoveryEvidence(
|
||
latestRun: LatestIssueRun,
|
||
): SuccessfulRunHandoffRecoveryEvidence | null {
|
||
if (!latestRun) return null;
|
||
|
||
const context = parseObject(latestRun.contextSnapshot);
|
||
const wakeReason = readNonEmptyString(context.wakeReason);
|
||
const handoffReason = readNonEmptyString(context.handoffReason);
|
||
const isSuccessfulRunHandoff =
|
||
wakeReason === FINISH_SUCCESSFUL_RUN_HANDOFF_REASON ||
|
||
handoffReason === SUCCESSFUL_RUN_MISSING_STATE_REASON ||
|
||
asBoolean(context.handoffRequired, false) === true;
|
||
if (!isSuccessfulRunHandoff) return null;
|
||
|
||
const handoffAttempt = asNumber(context.handoffAttempt, 1);
|
||
const maxHandoffAttempts = asNumber(
|
||
context.maxHandoffAttempts,
|
||
DEFAULT_MAX_SUCCESSFUL_RUN_HANDOFF_ATTEMPTS,
|
||
);
|
||
return {
|
||
sourceRunId:
|
||
readNonEmptyString(context.sourceRunId) ??
|
||
readNonEmptyString(context.resumeFromRunId),
|
||
correctiveRunId: latestRun.id,
|
||
missingDisposition:
|
||
readNonEmptyString(context.missingDisposition) ?? "clear_next_step",
|
||
handoffAttempt,
|
||
maxHandoffAttempts,
|
||
};
|
||
}
|
||
|
||
function isExhaustedSuccessfulRunHandoff(latestRun: LatestIssueRun) {
|
||
const evidence = successfulRunHandoffRecoveryEvidence(latestRun);
|
||
if (!evidence) return null;
|
||
if (evidence.handoffAttempt < evidence.maxHandoffAttempts)
|
||
return { ...evidence, exhausted: false };
|
||
return { ...evidence, exhausted: true };
|
||
}
|
||
|
||
function issueIdFromRunContext(contextSnapshot: unknown) {
|
||
const context = parseObject(contextSnapshot);
|
||
return (
|
||
readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId)
|
||
);
|
||
}
|
||
|
||
function issueIdFromWakePayload(payload: unknown) {
|
||
const parsed = parseObject(payload);
|
||
const nestedContext = parseObject(parsed[DEFERRED_WAKE_CONTEXT_KEY]);
|
||
return (
|
||
readNonEmptyString(parsed.issueId) ??
|
||
readNonEmptyString(nestedContext.issueId) ??
|
||
readNonEmptyString(nestedContext.taskId)
|
||
);
|
||
}
|
||
|
||
function issueUiLink(
|
||
issue: { identifier: string | null; id: string },
|
||
prefix: string,
|
||
) {
|
||
const label = issue.identifier ?? issue.id;
|
||
return `[${label}](/${prefix}/issues/${label})`;
|
||
}
|
||
|
||
function runUiLink(run: { id: string; agentId: string }, prefix: string) {
|
||
return `[${run.id}](/${prefix}/agents/${run.agentId}/runs/${run.id})`;
|
||
}
|
||
|
||
function agentUiLink(
|
||
agent: { id: string; name: string | null } | null,
|
||
prefix: string,
|
||
) {
|
||
if (!agent) return "unknown";
|
||
return `[${agent.name ?? agent.id}](/${prefix}/agents/${agent.id})`;
|
||
}
|
||
|
||
function formatIssueLinksForComment(
|
||
relations: Array<{ identifier?: string | null }>,
|
||
) {
|
||
const identifiers = [
|
||
...new Set(
|
||
relations
|
||
.map((relation) => relation.identifier)
|
||
.filter((identifier): identifier is string => Boolean(identifier)),
|
||
),
|
||
];
|
||
if (identifiers.length === 0) return "another open issue";
|
||
return identifiers
|
||
.slice(0, 5)
|
||
.map((identifier) => {
|
||
const prefix = identifier.split("-")[0] || "PAP";
|
||
return `[${identifier}](/${prefix}/issues/${identifier})`;
|
||
})
|
||
.join(", ");
|
||
}
|
||
|
||
function isStrandedIssueRecoveryIssue(
|
||
issue: Pick<typeof issues.$inferSelect, "originKind">,
|
||
) {
|
||
return isStrandedIssueRecoveryOriginKind(issue.originKind);
|
||
}
|
||
|
||
/**
|
||
* True when the issue's latest run was cancelled by a board operator (the
|
||
* board cancel route stamps the attribution; interrupt-by-comment uses the
|
||
* operator_interrupted error code). While such a run is the latest activity
|
||
* on an issue, recovery stands down entirely: the operator deliberately
|
||
* stopped the agent, and re-waking it — or escalating "stranding" — would
|
||
* fight the human. Any newer run or wake supersedes the exemption.
|
||
*/
|
||
function isOperatorCancelledRun(
|
||
latestRun: LatestIssueRun,
|
||
currentAgentId: string,
|
||
): boolean {
|
||
if (
|
||
latestRun?.agentId === currentAgentId &&
|
||
[
|
||
CHAT_CONTROL_RECOVERY_STOP_CODE,
|
||
CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,
|
||
].includes(latestRun.errorCode ?? "")
|
||
)
|
||
return true;
|
||
if (!latestRun || latestRun.status !== "cancelled") return false;
|
||
if (latestRun.errorCode === "operator_interrupted") return true;
|
||
const result = parseObject(latestRun.resultJson);
|
||
return (
|
||
result.cancelledByActorType === "user" ||
|
||
result.cancelledByActorType === "board"
|
||
);
|
||
}
|
||
|
||
function isUnsuccessfulTerminalIssueRun(latestRun: LatestIssueRun) {
|
||
return Boolean(
|
||
latestRun &&
|
||
UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.includes(
|
||
latestRun.status as (typeof UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES)[number],
|
||
),
|
||
);
|
||
}
|
||
|
||
function isSuccessfulInProgressContinuationRun(
|
||
latestRun: LatestIssueRun,
|
||
): latestRun is SuccessfulLatestIssueRun {
|
||
return latestRun?.status === "succeeded";
|
||
}
|
||
|
||
function isProductiveContinuationRun(latestRun: LatestIssueRun) {
|
||
return (
|
||
latestRun?.status === "succeeded" &&
|
||
(latestRun.livenessState === "advanced" ||
|
||
latestRun.livenessState === "completed" ||
|
||
latestRun.livenessState === "blocked" ||
|
||
latestRun.livenessState === "needs_followup")
|
||
);
|
||
}
|
||
|
||
function isRepeatedProductiveContinuationRecovery(
|
||
latestRun: SuccessfulLatestIssueRun,
|
||
) {
|
||
const latestContext = parseObject(latestRun.contextSnapshot);
|
||
return (
|
||
readNonEmptyString(latestContext.retryReason) ===
|
||
"issue_continuation_needed" &&
|
||
readNonEmptyString(latestContext.source) ===
|
||
"issue.productive_terminal_continuation_recovery" &&
|
||
isProductiveContinuationRun(latestRun)
|
||
);
|
||
}
|
||
|
||
export function recoveryService(
|
||
db: Db,
|
||
deps: {
|
||
enqueueWakeup: RecoveryWakeup;
|
||
scheduleRecoveryRetry?: (
|
||
runId: string,
|
||
) => Promise<typeof heartbeatRuns.$inferSelect | null>;
|
||
/** Settle retained explicit retry claims through the queue-first release policy. */
|
||
settleExplicitContinuationRetry?: (run: typeof heartbeatRuns.$inferSelect) => Promise<void>;
|
||
/**
|
||
* Whether a failed or interrupted run has consumed every bounded
|
||
* transient retry, so `scheduleRecoveryRetry` can no longer produce a
|
||
* successor for it. Lets the sweeper tell "no retry because the budget
|
||
* is spent" (escalate) from "no retry because something else owns the
|
||
* run" (leave alone).
|
||
*/
|
||
transientRetryBudgetSpent?: (
|
||
run: typeof heartbeatRuns.$inferSelect,
|
||
) => boolean;
|
||
liveRunExecutions?: Readonly<{ has(id: string): boolean }>;
|
||
beforeOrphanedRunTerminalWrite?: (runId: string) => Promise<void>;
|
||
},
|
||
) {
|
||
const issuesSvc = issueService(db);
|
||
const recoveryActionsSvc = issueRecoveryActionService(db);
|
||
const treeControlSvc = issueTreeControlService(db);
|
||
const budgets = budgetService(db);
|
||
let resolvedDependencyWakeBackstopCandidateCursor: string | null = null;
|
||
|
||
async function getAgent(agentId: string) {
|
||
return db
|
||
.select()
|
||
.from(agents)
|
||
.where(eq(agents.id, agentId))
|
||
.then((rows) => rows[0] ?? null);
|
||
}
|
||
|
||
async function isAgentInvokable(
|
||
agent: typeof agents.$inferSelect | null | undefined,
|
||
) {
|
||
return (await evaluateAgentInvokabilityFromDb(db, agent)).invokable;
|
||
}
|
||
|
||
async function getLatestIssueRun(
|
||
companyId: string,
|
||
issueId: string,
|
||
): Promise<LatestIssueRun> {
|
||
return db
|
||
.select({
|
||
id: heartbeatRuns.id,
|
||
agentId: heartbeatRuns.agentId,
|
||
status: heartbeatRuns.status,
|
||
error: heartbeatRuns.error,
|
||
errorCode: heartbeatRuns.errorCode,
|
||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||
livenessState: heartbeatRuns.livenessState,
|
||
resultJson: heartbeatRuns.resultJson,
|
||
startedAt: heartbeatRuns.startedAt,
|
||
createdAt: heartbeatRuns.createdAt,
|
||
})
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, companyId),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||
),
|
||
)
|
||
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id))
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
}
|
||
|
||
async function getLatestIssueRunForAgent(
|
||
companyId: string,
|
||
issueId: string,
|
||
agentId: string,
|
||
): Promise<LatestIssueRun> {
|
||
return db
|
||
.select({
|
||
id: heartbeatRuns.id,
|
||
agentId: heartbeatRuns.agentId,
|
||
status: heartbeatRuns.status,
|
||
error: heartbeatRuns.error,
|
||
errorCode: heartbeatRuns.errorCode,
|
||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||
livenessState: heartbeatRuns.livenessState,
|
||
resultJson: heartbeatRuns.resultJson,
|
||
startedAt: heartbeatRuns.startedAt,
|
||
createdAt: heartbeatRuns.createdAt,
|
||
})
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, companyId),
|
||
eq(heartbeatRuns.agentId, agentId),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||
),
|
||
)
|
||
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id))
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
}
|
||
|
||
async function summarizeRecentContinuationRetries(
|
||
companyId: string,
|
||
issueId: string,
|
||
agentId: string,
|
||
errorCodeToMatch: string | null,
|
||
since: Date | null = null,
|
||
) {
|
||
const rows = await db
|
||
.select({
|
||
id: heartbeatRuns.id,
|
||
status: heartbeatRuns.status,
|
||
errorCode: heartbeatRuns.errorCode,
|
||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||
finishedAt: heartbeatRuns.finishedAt,
|
||
})
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, companyId),
|
||
eq(heartbeatRuns.agentId, agentId),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||
...(since
|
||
? [
|
||
or(
|
||
gte(heartbeatRuns.createdAt, since),
|
||
gte(heartbeatRuns.finishedAt, since),
|
||
),
|
||
]
|
||
: []),
|
||
),
|
||
)
|
||
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id))
|
||
.limit(10);
|
||
|
||
let consecutive = 0;
|
||
let latestFinishedAt: Date | null = null;
|
||
for (const row of rows) {
|
||
const ctx = parseObject(row.contextSnapshot);
|
||
const retryReason = readNonEmptyString(ctx.retryReason);
|
||
if (retryReason !== "issue_continuation_needed") break;
|
||
if (
|
||
!UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.includes(
|
||
row.status as (typeof UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES)[number],
|
||
)
|
||
) {
|
||
break;
|
||
}
|
||
|
||
const rowErrorCode = readNonEmptyString(row.errorCode);
|
||
if (errorCodeToMatch !== rowErrorCode) {
|
||
break;
|
||
}
|
||
|
||
consecutive += 1;
|
||
if (latestFinishedAt === null) latestFinishedAt = row.finishedAt ?? null;
|
||
}
|
||
return { consecutive, latestFinishedAt };
|
||
}
|
||
|
||
async function hasActiveExecutionPath(
|
||
companyId: string,
|
||
issueId: string,
|
||
agentId?: string | null,
|
||
) {
|
||
const [run, deferredWake, nativeRecovery] = await Promise.all([
|
||
db
|
||
.select({ id: heartbeatRuns.id })
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, companyId),
|
||
inArray(heartbeatRuns.status, [
|
||
...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,
|
||
]),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||
agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`,
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null),
|
||
db
|
||
.select({ id: agentWakeupRequests.id })
|
||
.from(agentWakeupRequests)
|
||
.where(
|
||
and(
|
||
eq(agentWakeupRequests.companyId, companyId),
|
||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||
sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,
|
||
agentId ? eq(agentWakeupRequests.agentId, agentId) : sql`true`,
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null),
|
||
db
|
||
.select({ id: nativeRunFinalizations.runId })
|
||
.from(nativeRunFinalizations)
|
||
.innerJoin(
|
||
heartbeatRuns,
|
||
eq(heartbeatRuns.id, nativeRunFinalizations.runId),
|
||
)
|
||
.where(
|
||
and(
|
||
eq(nativeRunFinalizations.companyId, companyId),
|
||
eq(nativeRunFinalizations.issueId, issueId),
|
||
isNull(nativeRunFinalizations.resultId),
|
||
agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`,
|
||
or(
|
||
inArray(nativeRunFinalizations.recoveryState, [
|
||
"awaiting_evidence",
|
||
"awaiting_runner_reattach",
|
||
"resuming_session",
|
||
"bootstrap_incomplete",
|
||
]),
|
||
eq(nativeRunFinalizations.phase, "retryable_failure"),
|
||
),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null),
|
||
]);
|
||
|
||
return Boolean(run || deferredWake || nativeRecovery);
|
||
}
|
||
|
||
async function wasBlockedByAssigneeAgent(
|
||
companyId: string,
|
||
issueId: string,
|
||
agentId: string,
|
||
blockedTransitionAt: Date | null,
|
||
blockerIssueIds: string[],
|
||
) {
|
||
if (!blockedTransitionAt) return false;
|
||
const row = await db
|
||
.select({
|
||
actorType: activityLog.actorType,
|
||
actorId: activityLog.actorId,
|
||
agentId: activityLog.agentId,
|
||
runAgentId: heartbeatRuns.agentId,
|
||
details: activityLog.details,
|
||
})
|
||
.from(activityLog)
|
||
.leftJoin(heartbeatRuns, and(
|
||
eq(heartbeatRuns.id, activityLog.runId),
|
||
eq(heartbeatRuns.companyId, companyId),
|
||
or(eq(heartbeatRuns.issueId, issueId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`),
|
||
))
|
||
.where(
|
||
and(
|
||
eq(activityLog.companyId, companyId),
|
||
eq(activityLog.entityType, "issue"),
|
||
eq(activityLog.entityId, issueId),
|
||
eq(activityLog.action, "issue.updated"),
|
||
gte(activityLog.createdAt, new Date(blockedTransitionAt.getTime() - 5_000)),
|
||
sql`(
|
||
${activityLog.details} ?| array['status', 'toStatus', 'assigneeAgentId', 'assigneeUserId', 'blockedByIssueIds']
|
||
or (${activityLog.details}->'changes') ?| array['status', 'assigneeAgentId', 'assigneeUserId', 'blockedByIssueIds']
|
||
)`,
|
||
),
|
||
)
|
||
.orderBy(desc(activityLog.createdAt), desc(activityLog.id))
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
const details = row?.details ?? {};
|
||
const selfBlocked = row?.actorType === "agent" && row.agentId === agentId && details.status === "blocked";
|
||
const nativeSelfBlocked = row?.actorType === "system" &&
|
||
row.actorId === "native-status-committer" && row.runAgentId === agentId &&
|
||
details.source === "native_status_decision" && details.toStatus === "blocked";
|
||
if (!selfBlocked && !nativeSelfBlocked) return false;
|
||
const newlyCompletedBlocker = await db.select({ id: issues.id }).from(issues).where(and(
|
||
eq(issues.companyId, companyId),
|
||
inArray(issues.id, blockerIssueIds),
|
||
gt(issues.completedAt, blockedTransitionAt),
|
||
)).limit(1);
|
||
if (newlyCompletedBlocker.length > 0) return false;
|
||
const newlyFinalizedBlocker = await db.select({ id: workspaceOperations.id }).from(workspaceOperations)
|
||
.innerJoin(issues, and(eq(issues.companyId, companyId), inArray(issues.id, blockerIssueIds),
|
||
eq(issues.executionWorkspaceId, workspaceOperations.executionWorkspaceId),
|
||
eq(workspaceOperations.issueId, issues.id)))
|
||
.where(and(eq(workspaceOperations.companyId, companyId), eq(workspaceOperations.phase, "workspace_finalize"),
|
||
eq(workspaceOperations.status, "succeeded"), gt(workspaceOperations.finishedAt, blockedTransitionAt)))
|
||
.limit(1);
|
||
return newlyFinalizedBlocker.length === 0;
|
||
}
|
||
|
||
async function hasPendingWakeInteraction(companyId: string, issueId: string) {
|
||
return db
|
||
.select({ id: issueThreadInteractions.id })
|
||
.from(issueThreadInteractions)
|
||
.where(
|
||
and(
|
||
eq(issueThreadInteractions.companyId, companyId),
|
||
eq(issueThreadInteractions.issueId, issueId),
|
||
eq(issueThreadInteractions.status, "pending"),
|
||
inArray(issueThreadInteractions.continuationPolicy, [
|
||
"wake_assignee",
|
||
"wake_assignee_on_accept",
|
||
]),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => Boolean(rows[0]));
|
||
}
|
||
|
||
/**
|
||
* Pausing an agent does not turn an already committed passive response into
|
||
* stranded work. This only preserves the exact current wait; it grants no
|
||
* execution or presentation authority and does not repair historical state.
|
||
* Keep it separate from the broader monitor/delegated/legacy wait predicate.
|
||
*/
|
||
async function hasCurrentNativePassiveWait(
|
||
issue: typeof issues.$inferSelect,
|
||
latestRun: LatestIssueRun,
|
||
): Promise<boolean> {
|
||
if (
|
||
issue.status !== "in_progress" ||
|
||
latestRun?.status !== "succeeded" ||
|
||
latestRun.agentId !== issue.assigneeAgentId
|
||
)
|
||
return false;
|
||
const binding = {
|
||
companyId: issue.companyId,
|
||
issueId: issue.id,
|
||
runId: latestRun.id,
|
||
agentId: latestRun.agentId,
|
||
};
|
||
if (await hasCommittedNativePlanWait(db, binding)) return true;
|
||
const [receipt] = await db
|
||
.select({
|
||
run: heartbeatRuns,
|
||
resultJson: nativeRunResults.resultJson,
|
||
reason: statusDecisions.reasonCode,
|
||
})
|
||
.from(nativeRunFinalizations)
|
||
.innerJoin(
|
||
nativeRunResults,
|
||
and(
|
||
eq(nativeRunResults.id, nativeRunFinalizations.resultId),
|
||
eq(nativeRunResults.companyId, nativeRunFinalizations.companyId),
|
||
eq(nativeRunResults.issueId, nativeRunFinalizations.issueId),
|
||
eq(nativeRunResults.runId, nativeRunFinalizations.runId),
|
||
eq(nativeRunResults.schemaStatus, "accepted"),
|
||
),
|
||
)
|
||
.innerJoin(
|
||
heartbeatRuns,
|
||
and(
|
||
eq(heartbeatRuns.id, nativeRunResults.runId),
|
||
eq(heartbeatRuns.companyId, nativeRunResults.companyId),
|
||
eq(heartbeatRuns.nativeIssueId, nativeRunResults.issueId),
|
||
eq(
|
||
heartbeatRuns.completionContractId,
|
||
nativeRunResults.completionContractId,
|
||
),
|
||
eq(heartbeatRuns.agentId, binding.agentId),
|
||
eq(heartbeatRuns.runtimeMode, "native"),
|
||
eq(heartbeatRuns.status, "succeeded"),
|
||
),
|
||
)
|
||
.innerJoin(
|
||
statusDecisions,
|
||
and(
|
||
eq(statusDecisions.id, nativeRunFinalizations.decisionId),
|
||
eq(statusDecisions.assessmentId, nativeRunFinalizations.assessmentId),
|
||
eq(statusDecisions.companyId, binding.companyId),
|
||
eq(statusDecisions.issueId, binding.issueId),
|
||
eq(statusDecisions.runId, binding.runId),
|
||
eq(statusDecisions.applicationState, "applied"),
|
||
eq(statusDecisions.toStatus, "in_progress"),
|
||
inArray(statusDecisions.reasonCode, [
|
||
"board_response_waiting",
|
||
"external_chat_response_waiting",
|
||
]),
|
||
),
|
||
)
|
||
.innerJoin(
|
||
issues,
|
||
and(
|
||
eq(issues.id, binding.issueId),
|
||
eq(issues.companyId, binding.companyId),
|
||
eq(issues.assigneeAgentId, binding.agentId),
|
||
eq(issues.status, "in_progress"),
|
||
eq(issues.lastStatusDecisionId, statusDecisions.id),
|
||
isNull(issues.hiddenAt),
|
||
sql`coalesce(${issues.executionState}->>'status', '') <> 'pending'`,
|
||
),
|
||
)
|
||
.where(
|
||
and(
|
||
eq(nativeRunFinalizations.companyId, binding.companyId),
|
||
eq(nativeRunFinalizations.issueId, binding.issueId),
|
||
eq(nativeRunFinalizations.runId, binding.runId),
|
||
eq(nativeRunFinalizations.phase, "committed"),
|
||
),
|
||
)
|
||
.limit(1);
|
||
if (!receipt) return false;
|
||
const envelope = parseObject(receipt.resultJson);
|
||
const result = parseObject(envelope.result);
|
||
const terminal = parseObject(envelope.terminal);
|
||
const continuation = parseObject(result.continuation);
|
||
if (
|
||
result.schema !== "paperclip.run_result.v1" ||
|
||
result.reportedWorkDisposition !== "yielded" ||
|
||
continuation.kind !== "response_wake" ||
|
||
!readNonEmptyString(continuation.idempotencyKey) ||
|
||
terminal.runTerminalState !== "succeeded" ||
|
||
terminal.turnTerminalState !== "completed" ||
|
||
terminal.reportedWorkDisposition !== "yielded" ||
|
||
!Array.isArray(result.attentionRequests) ||
|
||
result.attentionRequests.length !== 0
|
||
)
|
||
return false;
|
||
if (receipt.reason === "board_response_waiting") {
|
||
return (
|
||
(await hasCommittedNativeBoardResponseWait(db, binding)) &&
|
||
(await readNativeBoardResponseWaitSource(db, binding)) !== null
|
||
);
|
||
}
|
||
|
||
const context = parseObject(receipt.run.contextSnapshot);
|
||
const ids = context.wakeCommentIds;
|
||
if (
|
||
!Array.isArray(ids) ||
|
||
ids.length === 0 ||
|
||
ids.length > 50 ||
|
||
ids.some(
|
||
(id) =>
|
||
typeof id !== "string" ||
|
||
!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(
|
||
id,
|
||
),
|
||
) ||
|
||
new Set(ids).size !== ids.length ||
|
||
!receipt.run.startedAt
|
||
)
|
||
return false;
|
||
const commentIds = ids as string[];
|
||
const sources = await db
|
||
.select({ id: issueComments.id })
|
||
.from(issueComments)
|
||
.where(
|
||
and(
|
||
eq(issueComments.companyId, binding.companyId),
|
||
eq(issueComments.issueId, binding.issueId),
|
||
inArray(issueComments.id, commentIds),
|
||
isNull(issueComments.createdByRunId),
|
||
isNull(issueComments.deletedAt),
|
||
sql`${issueComments.updatedAt} = ${issueComments.createdAt}`,
|
||
sql`${issueComments.createdAt} <= (select started_at from heartbeat_runs where id = ${binding.runId})`,
|
||
),
|
||
);
|
||
if (sources.length !== commentIds.length) return false;
|
||
const [newer, pendingInteraction, pendingApproval] = await Promise.all([
|
||
db
|
||
.select({ id: issueComments.id })
|
||
.from(issueComments)
|
||
.where(
|
||
and(
|
||
eq(issueComments.companyId, binding.companyId),
|
||
eq(issueComments.issueId, binding.issueId),
|
||
eq(issueComments.authorType, "user"),
|
||
isNull(issueComments.createdByRunId),
|
||
isNull(issueComments.deletedAt),
|
||
sql`(${issueComments.createdAt}, ${issueComments.id}) > (select created_at, id from issue_comments where id in (${sql.join(
|
||
commentIds.map((id) => sql`${id}::uuid`),
|
||
sql`, `,
|
||
)}) order by created_at desc, id desc limit 1)`,
|
||
),
|
||
)
|
||
.limit(1),
|
||
db
|
||
.select({ id: issueThreadInteractions.id })
|
||
.from(issueThreadInteractions)
|
||
.where(
|
||
and(
|
||
eq(issueThreadInteractions.companyId, binding.companyId),
|
||
eq(issueThreadInteractions.issueId, binding.issueId),
|
||
eq(issueThreadInteractions.status, "pending"),
|
||
),
|
||
)
|
||
.limit(1),
|
||
db
|
||
.select({ id: approvals.id })
|
||
.from(issueApprovals)
|
||
.innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId))
|
||
.where(
|
||
and(
|
||
eq(issueApprovals.companyId, binding.companyId),
|
||
eq(issueApprovals.issueId, binding.issueId),
|
||
eq(approvals.companyId, binding.companyId),
|
||
inArray(approvals.status, ["pending", "revision_requested"]),
|
||
),
|
||
)
|
||
.limit(1),
|
||
]);
|
||
if (newer.length || pendingInteraction.length || pendingApproval.length)
|
||
return false;
|
||
try {
|
||
await authorizeChatConversationForBoundRun(
|
||
db,
|
||
binding,
|
||
receipt.run.contextSnapshot,
|
||
"read",
|
||
);
|
||
return true;
|
||
} catch (error) {
|
||
if (
|
||
error instanceof Error &&
|
||
[
|
||
"paperclip_runner_chat_attachment_binding_denied",
|
||
"paperclip_runner_chat_attachment_destination_denied",
|
||
"paperclip_runner_chat_attachment_principal_denied",
|
||
].includes(error.message)
|
||
)
|
||
return false;
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async function hasPersistedDurableWaitPath(
|
||
issue: typeof issues.$inferSelect,
|
||
latestRun: LatestIssueRun,
|
||
) {
|
||
if (issue.monitorNextCheckAt) return true;
|
||
if (issue.status === "in_progress" && latestRun?.status === "succeeded" && latestRun.agentId === issue.assigneeAgentId &&
|
||
await hasCommittedNativePlanWait(db, { companyId: issue.companyId, issueId: issue.id, runId: latestRun.id, agentId: latestRun.agentId })) return true;
|
||
if (
|
||
issue.status === "in_progress" &&
|
||
latestRun?.status === "succeeded" &&
|
||
latestRun.agentId === issue.assigneeAgentId &&
|
||
(await hasCommittedNativeBoardResponseWait(db, {
|
||
companyId: issue.companyId,
|
||
issueId: issue.id,
|
||
runId: latestRun.id,
|
||
agentId: latestRun.agentId,
|
||
}))
|
||
)
|
||
return true;
|
||
|
||
// A provider thread owns the next wake for a successful external-chat
|
||
// turn. The Paperclip issue intentionally remains in progress so the next
|
||
// message can reuse it; that idle state is not stranded execution. Keep
|
||
// this scoped to the run that actually came from chat so an unrelated
|
||
// board/internal run on the same issue retains normal recovery semantics.
|
||
if (
|
||
issue.status === "in_progress" &&
|
||
latestRun?.status === "succeeded" &&
|
||
isExternalChatPresentationContext(latestRun.contextSnapshot)
|
||
) {
|
||
const activeConversation = await db
|
||
.select({ id: chatConversations.id })
|
||
.from(chatConversations)
|
||
.where(
|
||
and(
|
||
eq(chatConversations.companyId, issue.companyId),
|
||
eq(chatConversations.issueId, issue.id),
|
||
inArray(chatConversations.state, ["active", "waiting"]),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
if (activeConversation) return true;
|
||
}
|
||
|
||
return db
|
||
.select({ id: issueRelations.issueId })
|
||
.from(issueRelations)
|
||
.innerJoin(issues, eq(issueRelations.issueId, issues.id))
|
||
.where(
|
||
and(
|
||
eq(issueRelations.companyId, issue.companyId),
|
||
eq(issueRelations.relatedIssueId, issue.id),
|
||
eq(issueRelations.type, "blocks"),
|
||
eq(issues.companyId, issue.companyId),
|
||
notInArray(issues.status, ["done", "cancelled"]),
|
||
isNull(issues.hiddenAt),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => Boolean(rows[0]));
|
||
}
|
||
|
||
async function nativeBlockedUnblockAction(
|
||
issue: typeof issues.$inferSelect,
|
||
latestRun: LatestIssueRun,
|
||
database: Db = db,
|
||
) {
|
||
if (issue.status !== "in_progress" || latestRun?.status !== "succeeded")
|
||
return null;
|
||
// Consult the accepted, finalized result bound to this exact native run
|
||
// and contract, not model-looking fields in a legacy adapter summary.
|
||
const row = await database
|
||
.select({
|
||
resultJson: nativeRunResults.resultJson,
|
||
resultId: nativeRunResults.id,
|
||
resultSha256: nativeRunResults.canonicalSha256,
|
||
contractId: nativeRunResults.completionContractId,
|
||
decisionId: nativeRunFinalizations.decisionId,
|
||
assessmentId: nativeRunFinalizations.assessmentId,
|
||
finalizedAt: nativeRunFinalizations.updatedAt,
|
||
})
|
||
.from(nativeRunFinalizations)
|
||
.innerJoin(
|
||
nativeRunResults,
|
||
and(
|
||
eq(nativeRunResults.id, nativeRunFinalizations.resultId),
|
||
eq(nativeRunResults.companyId, nativeRunFinalizations.companyId),
|
||
eq(nativeRunResults.issueId, nativeRunFinalizations.issueId),
|
||
eq(nativeRunResults.runId, nativeRunFinalizations.runId),
|
||
),
|
||
)
|
||
.innerJoin(
|
||
heartbeatRuns,
|
||
and(
|
||
eq(heartbeatRuns.id, nativeRunResults.runId),
|
||
eq(heartbeatRuns.companyId, nativeRunResults.companyId),
|
||
eq(heartbeatRuns.nativeIssueId, nativeRunResults.issueId),
|
||
eq(
|
||
heartbeatRuns.completionContractId,
|
||
nativeRunResults.completionContractId,
|
||
),
|
||
),
|
||
)
|
||
.where(
|
||
and(
|
||
eq(nativeRunFinalizations.companyId, issue.companyId),
|
||
eq(nativeRunFinalizations.issueId, issue.id),
|
||
eq(nativeRunFinalizations.runId, latestRun.id),
|
||
eq(nativeRunFinalizations.phase, "committed"),
|
||
eq(nativeRunResults.schemaStatus, "accepted"),
|
||
eq(heartbeatRuns.runtimeMode, "native"),
|
||
eq(heartbeatRuns.status, "succeeded"),
|
||
eq(heartbeatRuns.agentId, latestRun.agentId),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
const envelope = parseObject(row?.resultJson);
|
||
const result = parseObject(envelope.result);
|
||
const terminal = parseObject(envelope.terminal);
|
||
const blocker = parseObject(result.blocker);
|
||
if (
|
||
result.schema !== "paperclip.run_result.v1" ||
|
||
result.reportedWorkDisposition !== "blocked" ||
|
||
terminal.runTerminalState !== "succeeded" ||
|
||
terminal.reportedWorkDisposition !== "blocked" ||
|
||
!["current_track", "task_wide"].includes(String(blocker.scope))
|
||
)
|
||
return null;
|
||
const action = readNonEmptyString(blocker.unblockAction);
|
||
return row && action ? { ...row, action } : null;
|
||
}
|
||
|
||
async function repairNativeBlockedWait(
|
||
issue: typeof issues.$inferSelect,
|
||
latestRun: NonNullable<LatestIssueRun>,
|
||
candidate: NonNullable<
|
||
Awaited<ReturnType<typeof nativeBlockedUnblockAction>>
|
||
>,
|
||
) {
|
||
const publications: ActivityPublication[] = [];
|
||
const postCommitActions: IssuePostCommitAction[] = [];
|
||
const repaired = await db.transaction(async (tx) => {
|
||
// Issue -> source evidence -> domain effects, matching ordinary comment
|
||
// admission and wake/queue editing. No provider I/O occurs under this lock.
|
||
const current = await tx
|
||
.select()
|
||
.from(issues)
|
||
.where(
|
||
and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id)),
|
||
)
|
||
.for("update")
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
if (
|
||
!current ||
|
||
current.status !== "in_progress" ||
|
||
current.statusVersion !== issue.statusVersion ||
|
||
current.lastStatusDecisionId !== issue.lastStatusDecisionId ||
|
||
current.assigneeAgentId !== issue.assigneeAgentId ||
|
||
current.assigneeUserId !== issue.assigneeUserId ||
|
||
current.executionRunId !== issue.executionRunId ||
|
||
current.checkoutRunId !== issue.checkoutRunId ||
|
||
current.updatedAt.getTime() !== issue.updatedAt.getTime() ||
|
||
current.hiddenAt ||
|
||
current.lastStatusDecisionId !== candidate.decisionId
|
||
)
|
||
return false;
|
||
|
||
const source = await tx
|
||
.select()
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, issue.companyId),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`,
|
||
),
|
||
)
|
||
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id))
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
if (
|
||
!source ||
|
||
source.id !== latestRun.id ||
|
||
source.agentId !== current.assigneeAgentId ||
|
||
source.status !== "succeeded" ||
|
||
JSON.stringify(source.contextSnapshot) !==
|
||
JSON.stringify(latestRun.contextSnapshot)
|
||
)
|
||
return false;
|
||
const proof = await nativeBlockedUnblockAction(
|
||
current,
|
||
source,
|
||
tx as unknown as Db,
|
||
);
|
||
if (
|
||
!proof ||
|
||
proof.resultId !== candidate.resultId ||
|
||
proof.resultSha256 !== candidate.resultSha256 ||
|
||
proof.contractId !== candidate.contractId ||
|
||
proof.action !== candidate.action ||
|
||
proof.decisionId !== candidate.decisionId ||
|
||
proof.assessmentId !== candidate.assessmentId ||
|
||
proof.finalizedAt.getTime() !== candidate.finalizedAt.getTime()
|
||
)
|
||
return false;
|
||
|
||
if (proof.assessmentId) {
|
||
const assessment = await tx
|
||
.select({ triggerKind: workAssessments.triggerKind })
|
||
.from(workAssessments)
|
||
.where(
|
||
and(
|
||
eq(workAssessments.id, proof.assessmentId),
|
||
eq(workAssessments.companyId, current.companyId),
|
||
eq(workAssessments.issueId, current.id),
|
||
eq(workAssessments.runId, source.id),
|
||
eq(workAssessments.resultId, proof.resultId),
|
||
eq(workAssessments.contractId, proof.contractId),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
// Attention routing can supersede this coordinator while retaining its
|
||
// blocked result. Only the original result assessment permits repair;
|
||
// a later independently authorized route remains handled but untouched.
|
||
if (assessment?.triggerKind !== "native_result") return false;
|
||
}
|
||
|
||
const context = parseObject(source.contextSnapshot);
|
||
const commentIds = [
|
||
...new Set(
|
||
[
|
||
readNonEmptyString(context.wakeCommentId),
|
||
...(Array.isArray(context.wakeCommentIds)
|
||
? context.wakeCommentIds.map(readNonEmptyString)
|
||
: []),
|
||
].filter((id): id is string => id !== null),
|
||
),
|
||
];
|
||
const comments = await tx
|
||
.select()
|
||
.from(issueComments)
|
||
.where(
|
||
and(
|
||
eq(issueComments.companyId, current.companyId),
|
||
eq(issueComments.issueId, current.id),
|
||
or(
|
||
commentIds.length
|
||
? inArray(issueComments.id, commentIds)
|
||
: sql`false`,
|
||
and(
|
||
or(
|
||
isNull(issueComments.createdByRunId),
|
||
not(eq(issueComments.createdByRunId, source.id)),
|
||
),
|
||
or(
|
||
gt(issueComments.createdAt, proof.finalizedAt),
|
||
gt(issueComments.updatedAt, proof.finalizedAt),
|
||
),
|
||
),
|
||
),
|
||
),
|
||
);
|
||
if (
|
||
comments.some(
|
||
(comment) =>
|
||
!commentIds.includes(comment.id) ||
|
||
comment.deletedAt ||
|
||
comment.updatedAt > proof.finalizedAt,
|
||
) ||
|
||
commentIds.some((id) => !comments.some((comment) => comment.id === id))
|
||
)
|
||
return false;
|
||
const paths = await collectDispositionRepairSourceState(
|
||
tx as unknown as Db,
|
||
{ issue: current },
|
||
);
|
||
if (paths.hasActiveExecutionPath || paths.hasDurableWaitingPath)
|
||
return false;
|
||
|
||
const updated = await issuesSvc.update(
|
||
current.id,
|
||
{
|
||
status: "blocked",
|
||
unblockDescriptor: { owner: "board", action: proof.action },
|
||
},
|
||
tx,
|
||
publications,
|
||
postCommitActions,
|
||
);
|
||
if (!updated) return false;
|
||
await issuesSvc.addComment(
|
||
current.id,
|
||
`The native run reported a blocker. The original request and assignee are preserved; no automatic continuation was started.\n\nUnblock request: ${proof.action}`,
|
||
{},
|
||
{
|
||
authorType: "system",
|
||
presentation: compactRecoveryPresentation(
|
||
"Waiting for the current request to be unblocked",
|
||
),
|
||
},
|
||
tx,
|
||
);
|
||
await logActivity(
|
||
tx as unknown as Db,
|
||
{
|
||
companyId: current.companyId,
|
||
actorType: "system",
|
||
actorId: "recovery",
|
||
action: "issue.updated",
|
||
entityType: "issue",
|
||
entityId: current.id,
|
||
details: {
|
||
source: "recovery.native_blocked_wait",
|
||
status: "blocked",
|
||
previousStatus: "in_progress",
|
||
latestRunId: source.id,
|
||
nativeResultId: proof.resultId,
|
||
completionContractId: proof.contractId,
|
||
},
|
||
},
|
||
publications,
|
||
);
|
||
return true;
|
||
});
|
||
if (repaired) {
|
||
for (const publication of publications) publishActivity(publication);
|
||
await executeIssuePostCommitActions(db, postCommitActions);
|
||
}
|
||
return repaired;
|
||
}
|
||
|
||
async function wasTodoHandedBackDuringOrAfterLatestRun(
|
||
issue: typeof issues.$inferSelect,
|
||
latestRun: LatestIssueRun,
|
||
) {
|
||
if (issue.status !== "todo" || latestRun?.status !== "succeeded")
|
||
return false;
|
||
const runBeganAt = latestRun.startedAt ?? latestRun.createdAt;
|
||
|
||
return db
|
||
.select({ id: issueRecoveryActions.id })
|
||
.from(issueRecoveryActions)
|
||
.where(
|
||
and(
|
||
eq(issueRecoveryActions.companyId, issue.companyId),
|
||
eq(issueRecoveryActions.sourceIssueId, issue.id),
|
||
eq(issueRecoveryActions.status, "resolved"),
|
||
eq(issueRecoveryActions.outcome, "handed_back"),
|
||
gte(issueRecoveryActions.resolvedAt, runBeganAt),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => Boolean(rows[0]));
|
||
}
|
||
|
||
async function hasQueuedIssueWake(
|
||
companyId: string,
|
||
issueId: string,
|
||
agentId?: string | null,
|
||
) {
|
||
return db
|
||
.select({ id: agentWakeupRequests.id })
|
||
.from(agentWakeupRequests)
|
||
.where(
|
||
and(
|
||
eq(agentWakeupRequests.companyId, companyId),
|
||
eq(agentWakeupRequests.status, "queued"),
|
||
sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,
|
||
agentId ? eq(agentWakeupRequests.agentId, agentId) : sql`true`,
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => Boolean(rows[0]));
|
||
}
|
||
|
||
async function getLatestAcceptedContinuationInteraction(
|
||
companyId: string,
|
||
issueId: string,
|
||
) {
|
||
return db
|
||
.select({
|
||
id: issueThreadInteractions.id,
|
||
kind: issueThreadInteractions.kind,
|
||
status: issueThreadInteractions.status,
|
||
continuationPolicy: issueThreadInteractions.continuationPolicy,
|
||
sourceRunId: issueThreadInteractions.sourceRunId,
|
||
resolvedAt: issueThreadInteractions.resolvedAt,
|
||
updatedAt: issueThreadInteractions.updatedAt,
|
||
})
|
||
.from(issueThreadInteractions)
|
||
.where(
|
||
and(
|
||
eq(issueThreadInteractions.companyId, companyId),
|
||
eq(issueThreadInteractions.issueId, issueId),
|
||
inArray(issueThreadInteractions.status, ["accepted", "answered"]),
|
||
inArray(issueThreadInteractions.continuationPolicy, [
|
||
"wake_assignee",
|
||
"wake_assignee_on_accept",
|
||
]),
|
||
),
|
||
)
|
||
.orderBy(
|
||
desc(
|
||
sql`coalesce(${issueThreadInteractions.resolvedAt}, ${issueThreadInteractions.updatedAt})`,
|
||
),
|
||
desc(issueThreadInteractions.id),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
}
|
||
|
||
async function hasSuccessfulIssueRunSince(
|
||
companyId: string,
|
||
issueId: string,
|
||
agentId: string,
|
||
since: Date,
|
||
interactionId?: string | null,
|
||
) {
|
||
return db
|
||
.select({ id: heartbeatRuns.id })
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, companyId),
|
||
eq(heartbeatRuns.agentId, agentId),
|
||
eq(heartbeatRuns.status, "succeeded"),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||
interactionId
|
||
? sql`${heartbeatRuns.contextSnapshot} ->> 'interactionId' = ${interactionId}`
|
||
: sql`true`,
|
||
or(
|
||
gte(heartbeatRuns.createdAt, since),
|
||
gte(heartbeatRuns.finishedAt, since),
|
||
),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => Boolean(rows[0]));
|
||
}
|
||
|
||
async function getLatestIssueRunSince(
|
||
companyId: string,
|
||
issueId: string,
|
||
agentId: string,
|
||
since: Date,
|
||
): Promise<LatestIssueRun> {
|
||
return db
|
||
.select({
|
||
id: heartbeatRuns.id,
|
||
agentId: heartbeatRuns.agentId,
|
||
status: heartbeatRuns.status,
|
||
error: heartbeatRuns.error,
|
||
errorCode: heartbeatRuns.errorCode,
|
||
contextSnapshot: heartbeatRuns.contextSnapshot,
|
||
livenessState: heartbeatRuns.livenessState,
|
||
resultJson: heartbeatRuns.resultJson,
|
||
startedAt: heartbeatRuns.startedAt,
|
||
createdAt: heartbeatRuns.createdAt,
|
||
})
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, companyId),
|
||
eq(heartbeatRuns.agentId, agentId),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||
or(
|
||
gte(heartbeatRuns.createdAt, since),
|
||
gte(heartbeatRuns.finishedAt, since),
|
||
),
|
||
),
|
||
)
|
||
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id))
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
}
|
||
|
||
// GGU-809: visible-progress signal for stranded-recovery escalation guard.
|
||
// Returns true if the assignee posted a comment, OR any attachment was added
|
||
// to the issue, within `windowMs`. Used to suppress false-positive recovery
|
||
// issues for batch workflows that genuinely advance every heartbeat.
|
||
async function hasRecentVisibleProgress(
|
||
companyId: string,
|
||
issueId: string,
|
||
assigneeAgentId: string,
|
||
windowMs: number,
|
||
) {
|
||
const since = new Date(Date.now() - windowMs);
|
||
const [comment, attachment] = await Promise.all([
|
||
db
|
||
.select({ id: issueComments.id })
|
||
.from(issueComments)
|
||
.where(
|
||
and(
|
||
eq(issueComments.companyId, companyId),
|
||
eq(issueComments.issueId, issueId),
|
||
eq(issueComments.authorAgentId, assigneeAgentId),
|
||
eq(issueComments.origin, "comment"),
|
||
gt(issueComments.createdAt, since),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null),
|
||
db
|
||
.select({ id: issueAttachments.id })
|
||
.from(issueAttachments)
|
||
.where(
|
||
and(
|
||
eq(issueAttachments.companyId, companyId),
|
||
eq(issueAttachments.issueId, issueId),
|
||
gt(issueAttachments.createdAt, since),
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null),
|
||
]);
|
||
return Boolean(comment || attachment);
|
||
}
|
||
|
||
async function enqueueStrandedIssueRecovery(input: {
|
||
issueId: string;
|
||
agentId: string;
|
||
reason:
|
||
| "issue_assignment_recovery"
|
||
| "issue_continuation_needed"
|
||
| typeof EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON;
|
||
retryReason:
|
||
| "assignment_recovery"
|
||
| "issue_continuation_needed"
|
||
| typeof EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON;
|
||
source: string;
|
||
retryOfRunId?: string | null;
|
||
extraContext?: Record<string, unknown>;
|
||
/**
|
||
* Out-parameter: set `retryExhausted` when no successor was queued
|
||
* because the failed predecessor has spent its bounded transient retry
|
||
* budget. A null return without it means another authority owns the
|
||
* run (native runtime, reconciliation) and the caller must leave it.
|
||
*/
|
||
outcome?: { retryExhausted?: boolean };
|
||
}) {
|
||
if (input.retryOfRunId) {
|
||
const [predecessor] = await db
|
||
.select()
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.id, input.retryOfRunId),
|
||
eq(heartbeatRuns.agentId, input.agentId),
|
||
),
|
||
);
|
||
if (
|
||
predecessor &&
|
||
["failed", "timed_out", "interrupted", "cancelled"].includes(
|
||
predecessor.status,
|
||
)
|
||
) {
|
||
// Failure recovery shares the durable incident budget and delay. It
|
||
// cannot fall through into the productive-work continuation queue.
|
||
if (predecessor.runtimeMode === "native") return null;
|
||
if (await legacyExecutionNeedsReconciliationWithEvidence(db, predecessor)) {
|
||
await terminalizeLegacyExecution({
|
||
db,
|
||
run: predecessor,
|
||
status: predecessor.status,
|
||
});
|
||
return null;
|
||
}
|
||
if (deps.scheduleRecoveryRetry) {
|
||
const retry = await deps.scheduleRecoveryRetry(predecessor.id);
|
||
if (retry) return retry;
|
||
// A spent budget is the one "no retry" the sweeper must not wait
|
||
// out: nothing else will ever queue a successor for this run, so
|
||
// it reports the exhaustion instead of leaving the issue with no
|
||
// live path (2026-09-25: three deploy restarts in a row spent the
|
||
// budget and the issue sat in_progress with no run, unescalated).
|
||
if (input.outcome && deps.transientRetryBudgetSpent?.(predecessor)) {
|
||
input.outcome.retryExhausted = true;
|
||
}
|
||
return null;
|
||
}
|
||
return null;
|
||
}
|
||
}
|
||
if (
|
||
input.retryOfRunId &&
|
||
input.source === "issue.productive_terminal_continuation_recovery"
|
||
) {
|
||
const [source] = await db
|
||
.select({
|
||
companyId: heartbeatRuns.companyId,
|
||
agentId: heartbeatRuns.agentId,
|
||
})
|
||
.from(heartbeatRuns)
|
||
.where(eq(heartbeatRuns.id, input.retryOfRunId))
|
||
.limit(1);
|
||
if (
|
||
!source ||
|
||
(source.agentId === input.agentId &&
|
||
(
|
||
await readChatControlRecoveryStop(db, {
|
||
...source,
|
||
issueId: input.issueId,
|
||
sourceRunId: input.retryOfRunId,
|
||
})
|
||
).kind !== "clear")
|
||
)
|
||
return null;
|
||
}
|
||
const queued = await deps.enqueueWakeup(input.agentId, {
|
||
source: "automation",
|
||
triggerDetail: "system",
|
||
reason: input.reason,
|
||
// The sweep can combine an old in-progress issue snapshot with a newer
|
||
// successful run. Validate eligibility under the enqueue issue lock so
|
||
// completion or reassignment cannot create a redundant continuation.
|
||
...(input.source === "issue.productive_terminal_continuation_recovery"
|
||
? {
|
||
issueStateGuard: {
|
||
statuses: ["in_progress"],
|
||
assigneeAgentId: input.agentId,
|
||
},
|
||
}
|
||
: {}),
|
||
payload: withRecoveryContext(
|
||
{
|
||
issueId: input.issueId,
|
||
...(input.retryOfRunId ? { retryOfRunId: input.retryOfRunId } : {}),
|
||
...(input.extraContext ?? {}),
|
||
},
|
||
"normal_model",
|
||
),
|
||
requestedByActorType: "system",
|
||
requestedByActorId: null,
|
||
contextSnapshot: withRecoveryContext(
|
||
{
|
||
issueId: input.issueId,
|
||
taskId: input.issueId,
|
||
wakeReason: input.reason,
|
||
retryReason: input.retryReason,
|
||
source: input.source,
|
||
...(input.retryOfRunId ? { retryOfRunId: input.retryOfRunId } : {}),
|
||
...(input.extraContext ?? {}),
|
||
},
|
||
"normal_model",
|
||
),
|
||
});
|
||
|
||
if (queued && input.retryOfRunId) {
|
||
return db
|
||
.update(heartbeatRuns)
|
||
.set({
|
||
retryOfRunId: input.retryOfRunId,
|
||
updatedAt: new Date(),
|
||
})
|
||
.where(eq(heartbeatRuns.id, queued.id))
|
||
.returning()
|
||
.then((rows) => rows[0] ?? queued);
|
||
}
|
||
|
||
return queued;
|
||
}
|
||
|
||
async function enqueueInitialAssignedTodoDispatch(
|
||
issue: typeof issues.$inferSelect,
|
||
agentId: string,
|
||
) {
|
||
return deps.enqueueWakeup(agentId, {
|
||
source: "assignment",
|
||
triggerDetail: "system",
|
||
reason: "issue_assigned",
|
||
payload: withRecoveryContext(
|
||
{
|
||
issueId: issue.id,
|
||
mutation: "assigned_todo_liveness_dispatch",
|
||
},
|
||
"normal_model",
|
||
),
|
||
requestedByActorType: "system",
|
||
requestedByActorId: null,
|
||
contextSnapshot: withRecoveryContext(
|
||
{
|
||
issueId: issue.id,
|
||
taskId: issue.id,
|
||
wakeReason: "issue_assigned",
|
||
source: "issue.assigned_todo_liveness_dispatch",
|
||
},
|
||
"normal_model",
|
||
),
|
||
});
|
||
}
|
||
|
||
// The onboarding first task (origin `onboarding_first_task`) is created with
|
||
// its greeting pre-seeded and *no* assignment wake on purpose: the product
|
||
// contract is that nothing runs until the user types. Until a user-authored
|
||
// comment exists on it, the issue is intentionally idle rather than stranded.
|
||
async function isOnboardingFirstTaskAwaitingUser(
|
||
issue: typeof issues.$inferSelect,
|
||
) {
|
||
if (issue.originKind !== ONBOARDING_FIRST_TASK_ORIGIN_KIND) return false;
|
||
const [result] = await db.execute<{ awaiting: boolean }>(sql`SELECT
|
||
${onboardingFirstTaskAwaitingUserSql(sql`${issue.companyId}`, sql`${issue.id}`, sql`${issue.originKind}`)} AS awaiting`);
|
||
return result?.awaiting === true;
|
||
}
|
||
|
||
async function isInvocationBudgetBlocked(
|
||
issue: typeof issues.$inferSelect,
|
||
agentId: string,
|
||
) {
|
||
const budgetBlock = await budgets.getInvocationBlock(
|
||
issue.companyId,
|
||
agentId,
|
||
{
|
||
issueId: issue.id,
|
||
projectId: issue.projectId,
|
||
},
|
||
);
|
||
return Boolean(budgetBlock);
|
||
}
|
||
|
||
async function reconcileUnassignedBlockingIssues() {
|
||
const candidates = await db
|
||
.select({
|
||
id: issues.id,
|
||
companyId: issues.companyId,
|
||
identifier: issues.identifier,
|
||
status: issues.status,
|
||
createdByAgentId: issues.createdByAgentId,
|
||
})
|
||
.from(issueRelations)
|
||
.innerJoin(issues, eq(issueRelations.issueId, issues.id))
|
||
.where(
|
||
and(
|
||
eq(issueRelations.type, "blocks"),
|
||
inArray(issues.status, ["todo", "blocked"]),
|
||
isNull(issues.assigneeAgentId),
|
||
isNull(issues.assigneeUserId),
|
||
sql`${issues.createdByAgentId} is not null`,
|
||
sql`exists (
|
||
select 1
|
||
from issues blocked_issue
|
||
where blocked_issue.id = ${issueRelations.relatedIssueId}
|
||
and blocked_issue.company_id = ${issues.companyId}
|
||
and blocked_issue.status not in ('done', 'cancelled')
|
||
)`,
|
||
),
|
||
);
|
||
|
||
let assigned = 0;
|
||
let skipped = 0;
|
||
const issueIds: string[] = [];
|
||
const seen = new Set<string>();
|
||
|
||
for (const candidate of candidates) {
|
||
if (seen.has(candidate.id)) continue;
|
||
seen.add(candidate.id);
|
||
|
||
const creatorAgentId = candidate.createdByAgentId;
|
||
if (!creatorAgentId) {
|
||
skipped += 1;
|
||
continue;
|
||
}
|
||
const creatorAgent = await getAgent(creatorAgentId);
|
||
if (
|
||
!creatorAgent ||
|
||
creatorAgent.companyId !== candidate.companyId ||
|
||
!(await isAgentInvokable(creatorAgent))
|
||
) {
|
||
skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const relations = await issuesSvc.getRelationSummaries(candidate.id);
|
||
const blockingLinks = formatIssueLinksForComment(relations.blocks);
|
||
const updated = await issuesSvc.update(candidate.id, {
|
||
assigneeAgentId: creatorAgent.id,
|
||
assigneeUserId: null,
|
||
});
|
||
if (!updated) {
|
||
skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
await issuesSvc.addComment(
|
||
candidate.id,
|
||
[
|
||
"## Assigned Orphan Blocker",
|
||
"",
|
||
`Paperclip found this issue is blocking ${blockingLinks} but had no assignee, so no heartbeat could pick it up.`,
|
||
"",
|
||
"- Assigned it back to the agent that created the blocker.",
|
||
"- Next action: resolve this blocker or reassign it to the right owner.",
|
||
].join("\n"),
|
||
{},
|
||
);
|
||
|
||
await logActivity(db, {
|
||
companyId: candidate.companyId,
|
||
actorType: "system",
|
||
actorId: "system",
|
||
agentId: null,
|
||
runId: null,
|
||
action: "issue.updated",
|
||
entityType: "issue",
|
||
entityId: candidate.id,
|
||
details: {
|
||
identifier: candidate.identifier,
|
||
assigneeAgentId: creatorAgent.id,
|
||
source: "recovery.reconcile_unassigned_blocking_issue",
|
||
},
|
||
});
|
||
|
||
const queued = await deps.enqueueWakeup(creatorAgent.id, {
|
||
source: "automation",
|
||
triggerDetail: "system",
|
||
reason: "issue_assigned",
|
||
payload: withRecoveryContext(
|
||
{
|
||
issueId: candidate.id,
|
||
mutation: "unassigned_blocker_recovery",
|
||
},
|
||
"normal_model",
|
||
),
|
||
requestedByActorType: "system",
|
||
requestedByActorId: null,
|
||
contextSnapshot: withRecoveryContext(
|
||
{
|
||
issueId: candidate.id,
|
||
taskId: candidate.id,
|
||
wakeReason: "issue_assigned",
|
||
source: "issue.unassigned_blocker_recovery",
|
||
},
|
||
"normal_model",
|
||
),
|
||
});
|
||
|
||
if (queued) {
|
||
assigned += 1;
|
||
issueIds.push(candidate.id);
|
||
} else {
|
||
skipped += 1;
|
||
}
|
||
}
|
||
|
||
return { assigned, skipped, issueIds };
|
||
}
|
||
|
||
async function getCompanyIssuePrefix(companyId: string) {
|
||
return db
|
||
.select({ issuePrefix: companies.issuePrefix })
|
||
.from(companies)
|
||
.where(eq(companies.id, companyId))
|
||
.then((rows) => rows[0]?.issuePrefix ?? "PAP");
|
||
}
|
||
|
||
const watchdog = createActiveRunWatchdog(db, {
|
||
suspicionThresholdMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS,
|
||
criticalThresholdMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS,
|
||
continueRearmMs: ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS,
|
||
});
|
||
|
||
async function buildRunOutputSilence(
|
||
run: Pick<
|
||
typeof heartbeatRuns.$inferSelect,
|
||
| "id"
|
||
| "companyId"
|
||
| "status"
|
||
| "lastOutputAt"
|
||
| "lastOutputSeq"
|
||
| "lastOutputStream"
|
||
| "processStartedAt"
|
||
| "startedAt"
|
||
| "createdAt"
|
||
>,
|
||
now = new Date(),
|
||
): Promise<RunOutputSilenceSummary> {
|
||
return watchdog.buildRunOutputSilence(run, now);
|
||
}
|
||
|
||
async function appendRecoveryRunEvent(
|
||
run: typeof heartbeatRuns.$inferSelect,
|
||
event: {
|
||
level: "info" | "warn" | "error";
|
||
message: string;
|
||
payload?: Record<string, unknown>;
|
||
},
|
||
) {
|
||
await appendHeartbeatRunEvent(db, {
|
||
companyId: run.companyId,
|
||
runId: run.id,
|
||
agentId: run.agentId,
|
||
eventType: "lifecycle",
|
||
stream: "system",
|
||
level: event.level,
|
||
message: event.message,
|
||
payload: event.payload ?? null,
|
||
});
|
||
}
|
||
|
||
async function scanSilentActiveRuns(opts?: {
|
||
now?: Date;
|
||
companyId?: string;
|
||
issueCreatedAtGte?: Date | null;
|
||
}) {
|
||
return watchdog.scanSilentActiveRuns(opts);
|
||
}
|
||
|
||
async function recordWatchdogDecision(input: {
|
||
runId: string;
|
||
actor: WatchdogDecisionActor;
|
||
decision: "snooze" | "continue" | "dismissed_false_positive";
|
||
evaluationIssueId?: string | null;
|
||
reason?: string | null;
|
||
snoozedUntil?: Date | null;
|
||
createdByRunId?: string | null;
|
||
now?: Date;
|
||
}) {
|
||
const [run] = await db
|
||
.select({ companyId: heartbeatRuns.companyId })
|
||
.from(heartbeatRuns)
|
||
.where(eq(heartbeatRuns.id, input.runId))
|
||
.limit(1);
|
||
if (!run) throw notFound("Heartbeat run not found");
|
||
try {
|
||
return await watchdog.recordWatchdogDecision({
|
||
...input,
|
||
companyId: run.companyId,
|
||
});
|
||
} catch (error) {
|
||
if (!(error instanceof WatchdogDecisionApplicationError)) throw error;
|
||
if (
|
||
error.code === "run_not_found" ||
|
||
error.code === "evaluation_issue_not_found"
|
||
) {
|
||
throw notFound(error.message);
|
||
}
|
||
throw forbidden(error.message);
|
||
}
|
||
}
|
||
|
||
function isStrandedIssueRecoveryIssue(issue: typeof issues.$inferSelect) {
|
||
return issue.originKind === STRANDED_ISSUE_RECOVERY_ORIGIN_KIND;
|
||
}
|
||
|
||
async function buildNestedStrandedRecoveryLine(
|
||
issue: typeof issues.$inferSelect,
|
||
prefix: string,
|
||
) {
|
||
const sourceIssueId = readNonEmptyString(issue.originId);
|
||
const sourceIssue = sourceIssueId
|
||
? await db
|
||
.select({ id: issues.id, identifier: issues.identifier })
|
||
.from(issues)
|
||
.where(
|
||
and(
|
||
eq(issues.companyId, issue.companyId),
|
||
eq(issues.id, sourceIssueId),
|
||
),
|
||
)
|
||
.then((rows) => rows[0] ?? null)
|
||
: null;
|
||
const sourceLine = sourceIssue
|
||
? `- Original source issue: ${issueUiLink(sourceIssue, prefix)}`
|
||
: sourceIssueId
|
||
? `- Original source issue: \`${sourceIssueId}\``
|
||
: "- Original source issue: unknown";
|
||
|
||
return [
|
||
"",
|
||
"- Nested recovery: suppressed because this issue is already a `stranded_issue_recovery` issue.",
|
||
sourceLine,
|
||
"- Next action: the assigned recovery owner or board operator should fix the runtime/adapter problem, resolve or reassign the original source issue, then mark this recovery issue done or cancelled.",
|
||
].join("\n");
|
||
}
|
||
|
||
function resolveStrandedRecoveryRouting(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
latestRun: LatestIssueRun;
|
||
}) {
|
||
const originalAgentId =
|
||
input.issue.assigneeAgentId ?? input.latestRun?.agentId ?? null;
|
||
return {
|
||
returnOwnerAgentId: originalAgentId,
|
||
};
|
||
}
|
||
|
||
function strandedRecoveryActionKind(cause: StrandedRecoveryCause) {
|
||
return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON
|
||
? ("missing_disposition" as const)
|
||
: cause === "deliberate_wait_without_target"
|
||
? ("deliberate_wait_without_target" as const)
|
||
: cause === "workspace_validation_failed"
|
||
? ("workspace_validation" as const)
|
||
: cause === "configuration_incomplete"
|
||
? ("configuration_validation" as const)
|
||
: ("stranded_assigned_issue" as const);
|
||
}
|
||
|
||
function strandedRecoveryActionFingerprint(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
recoveryCause: StrandedRecoveryCause;
|
||
latestRun: LatestIssueRun;
|
||
}) {
|
||
if (input.recoveryCause === "workspace_validation_failed") {
|
||
const workspaceFingerprint = readWorkspaceValidationFingerprint(
|
||
input.latestRun,
|
||
);
|
||
if (workspaceFingerprint) {
|
||
return [
|
||
"source_scoped_recovery",
|
||
input.issue.companyId,
|
||
input.issue.id,
|
||
input.recoveryCause,
|
||
workspaceFingerprint,
|
||
].join(":");
|
||
}
|
||
}
|
||
// A configuration-incomplete failure that carries a stable identity (for
|
||
// example an unresolved workspace base ref) dedupes per that identity, so a
|
||
// different requested ref makes a new recovery action while the same ref
|
||
// reuses one. Configuration gaps with no fingerprint fall back to the
|
||
// issue-and-cause scope below.
|
||
if (input.recoveryCause === "configuration_incomplete") {
|
||
const configurationFingerprint = readConfigurationIncompleteFingerprint(
|
||
input.latestRun,
|
||
);
|
||
if (configurationFingerprint) {
|
||
return [
|
||
"source_scoped_recovery",
|
||
input.issue.companyId,
|
||
input.issue.id,
|
||
input.recoveryCause,
|
||
configurationFingerprint,
|
||
].join(":");
|
||
}
|
||
}
|
||
return [
|
||
"source_scoped_recovery",
|
||
input.issue.companyId,
|
||
input.issue.id,
|
||
input.recoveryCause,
|
||
].join(":");
|
||
}
|
||
|
||
function buildStrandedRecoveryActionEvidence(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
latestRun: LatestIssueRun;
|
||
previousStatus: StrandedPreviousStatus;
|
||
recoveryCause: StrandedRecoveryCause;
|
||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||
}) {
|
||
const context = parseObject(input.latestRun?.contextSnapshot);
|
||
const workspaceValidation =
|
||
input.recoveryCause === "workspace_validation_failed"
|
||
? readWorkspaceValidationPayload(input.latestRun)
|
||
: null;
|
||
return {
|
||
sourceIssueId: input.issue.id,
|
||
sourceIdentifier: input.issue.identifier,
|
||
previousStatus: input.previousStatus,
|
||
latestIssueStatus: input.issue.status,
|
||
latestRunId: input.latestRun?.id ?? null,
|
||
latestRunStatus: input.latestRun?.status ?? null,
|
||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||
retryReason: readNonEmptyString(context.retryReason) ?? null,
|
||
recoveryCause: input.recoveryCause,
|
||
sourceRunId: input.successfulRunHandoffEvidence?.sourceRunId ?? null,
|
||
correctiveRunId:
|
||
input.successfulRunHandoffEvidence?.correctiveRunId ?? null,
|
||
missingDisposition:
|
||
input.successfulRunHandoffEvidence?.missingDisposition ?? null,
|
||
handoffAttempt:
|
||
input.successfulRunHandoffEvidence?.handoffAttempt ?? null,
|
||
maxHandoffAttempts:
|
||
input.successfulRunHandoffEvidence?.maxHandoffAttempts ?? null,
|
||
...(workspaceValidation ? { workspaceValidation } : {}),
|
||
};
|
||
}
|
||
|
||
async function ensureSourceScopedStrandedRecoveryAction(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
latestRun: LatestIssueRun;
|
||
previousStatus: StrandedPreviousStatus;
|
||
recoveryCause?: StrandedRecoveryCause;
|
||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||
}) {
|
||
const recoveryCause = resolveStrandedRecoveryCause(
|
||
input.latestRun,
|
||
input.recoveryCause,
|
||
);
|
||
const routing = resolveStrandedRecoveryRouting({
|
||
issue: input.issue,
|
||
latestRun: input.latestRun,
|
||
});
|
||
const isProviderQuotaWait = recoveryCause === "provider_quota";
|
||
const now = new Date();
|
||
const action = await recoveryActionsSvc.upsertSourceScoped({
|
||
companyId: input.issue.companyId,
|
||
sourceIssueId: input.issue.id,
|
||
// A configuration-incomplete failure carries a per-identity fingerprint
|
||
// (for example the unresolved workspace base ref). A different ref is a
|
||
// distinct blocker, so it must get a new recovery action and notify the
|
||
// operator, not overwrite the active action of the prior ref.
|
||
supersedeOnIdentityChange: recoveryCause === "configuration_incomplete",
|
||
preserveExistingOwner: true,
|
||
kind: strandedRecoveryActionKind(recoveryCause),
|
||
ownerType: isProviderQuotaWait ? "system" : "board",
|
||
ownerAgentId: null,
|
||
ownerUserId: null,
|
||
previousOwnerAgentId: input.issue.assigneeAgentId,
|
||
returnOwnerAgentId: routing.returnOwnerAgentId,
|
||
cause: recoveryCause,
|
||
fingerprint: strandedRecoveryActionFingerprint({
|
||
issue: input.issue,
|
||
recoveryCause,
|
||
latestRun: input.latestRun,
|
||
}),
|
||
evidence: {
|
||
...buildStrandedRecoveryActionEvidence({
|
||
issue: input.issue,
|
||
latestRun: input.latestRun,
|
||
previousStatus: input.previousStatus,
|
||
recoveryCause,
|
||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||
}),
|
||
failureSummary:
|
||
summarizeRunFailureForIssueComment(input.latestRun)?.trim() ?? null,
|
||
},
|
||
evidenceOnCreate: isProviderQuotaWait
|
||
? {}
|
||
: { routingPolicy: STRANDED_BOARD_ESCALATION_POLICY },
|
||
nextAction:
|
||
recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
|
||
? "Board operator: inspect the run evidence, then explicitly choose a valid issue disposition, retry the original owner, reassign, or intentionally resolve the task."
|
||
: recoveryCause === "process_lost"
|
||
? "Board operator: inspect the retry history, then explicitly retry the original owner, reassign, or intentionally resolve the task."
|
||
: recoveryCause === "provider_quota"
|
||
? "Wait for provider quota recovery, then retry the original assignee; do not wake a takeover owner."
|
||
: recoveryCause === "codex_output_inactivity_monitor"
|
||
? "Board operator: inspect the inactivity evidence, then explicitly retry the original owner, reassign, or intentionally resolve the task."
|
||
: recoveryCause === "workspace_validation_failed"
|
||
? readWorkspaceValidationPayload(input.latestRun)?.reason ===
|
||
"git_worktree_branch_incoherence"
|
||
? "Board operator: repair the source task git worktree branch incoherence or choose a new execution workspace, then explicitly retry or reassign."
|
||
: readWorkspaceValidationPayload(input.latestRun)
|
||
?.reason ===
|
||
"git_worktree_base_materialization_failed"
|
||
? "Board operator: repair the project workspace repository URL or clone access, or configure a local checkout cwd, then explicitly retry or reassign."
|
||
: "Board operator: repair the source task workspace link, project workspace cwd, or git checkout, then explicitly retry or reassign."
|
||
: recoveryCause === "configuration_incomplete"
|
||
? readConfigurationIncompletePayload(input.latestRun)?.reason === "workspace_base_ref_unresolved"
|
||
? "Check the starting branch and repository access, then repair the task’s branch and retry the original assignee."
|
||
: readConfigurationIncompletePayload(input.latestRun)?.reason === "ai_connection_unavailable"
|
||
? "Reconnect the selected AI account or choose an available connection, then continue the task."
|
||
: readConfigurationIncompletePayload(input.latestRun)
|
||
?.reason === SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON
|
||
? `Board operator: the sandbox provider plugin named in the run failure is not ready; ${sandboxProviderPluginRemedy(
|
||
readNonEmptyString(
|
||
readConfigurationIncompletePayload(input.latestRun)
|
||
?.pluginStatus,
|
||
) ?? "error",
|
||
)}, then explicitly retry the original owner or reassign.`
|
||
: "Board operator: bind the missing secret(s) named in the run failure, then explicitly retry the original owner or reassign."
|
||
: recoveryCause === "execution_review_participant_recovery"
|
||
? "Board operator: repair the failed review participant path, restore a live reviewer, explicitly reassign, or record an intentional resolution."
|
||
: "Board operator: inspect the evidence, repair the runtime if appropriate, then explicitly retry the original owner, reassign, or intentionally resolve the task.",
|
||
wakePolicy: isProviderQuotaWait
|
||
? {
|
||
type: "monitor_only",
|
||
reason: recoveryCause,
|
||
}
|
||
: {
|
||
type: "board_escalation",
|
||
reason: recoveryCause,
|
||
preservesSourceAssignee: true,
|
||
},
|
||
monitorPolicy: isProviderQuotaWait
|
||
? { type: "wait_recovery", retryAgentId: routing.returnOwnerAgentId }
|
||
: null,
|
||
maxAttempts: null,
|
||
lastAttemptAt: now,
|
||
});
|
||
|
||
return action;
|
||
}
|
||
|
||
function readProviderQuotaRetryAt(latestRun: LatestIssueRun, now: Date) {
|
||
const result = parseObject(latestRun?.resultJson);
|
||
const context = parseObject(latestRun?.contextSnapshot);
|
||
const raw =
|
||
result.providerQuotaRetryNotBefore ??
|
||
result.retryNotBefore ??
|
||
result.transientRetryNotBefore ??
|
||
context.providerQuotaRetryNotBefore ??
|
||
context.transientRetryNotBefore;
|
||
if (
|
||
typeof raw === "string" ||
|
||
typeof raw === "number" ||
|
||
raw instanceof Date
|
||
) {
|
||
const parsed = new Date(raw);
|
||
if (!Number.isNaN(parsed.getTime()) && parsed.getTime() > now.getTime())
|
||
return parsed;
|
||
}
|
||
return new Date(now.getTime() + PROVIDER_QUOTA_RECOVERY_DEFAULT_BACKOFF_MS);
|
||
}
|
||
|
||
async function ensureProviderQuotaWaitRecoveryMonitor(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
latestRun: LatestIssueRun;
|
||
actionId: string;
|
||
agentId: string;
|
||
}) {
|
||
const existing = await db
|
||
.select()
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, input.issue.companyId),
|
||
eq(heartbeatRuns.agentId, input.agentId),
|
||
eq(heartbeatRuns.status, "scheduled_retry"),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${input.issue.id}`,
|
||
),
|
||
)
|
||
.orderBy(desc(heartbeatRuns.scheduledRetryAt))
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
if (existing) return existing;
|
||
|
||
const now = new Date();
|
||
const retryAt = readProviderQuotaRetryAt(input.latestRun, now);
|
||
return db.transaction(async (tx) => {
|
||
const wakeup = await tx
|
||
.insert(agentWakeupRequests)
|
||
.values({
|
||
companyId: input.issue.companyId,
|
||
agentId: input.agentId,
|
||
source: "automation",
|
||
triggerDetail: "system",
|
||
reason: "provider_quota_recovery",
|
||
payload: withRecoveryContext(
|
||
{
|
||
issueId: input.issue.id,
|
||
retryOfRunId: input.latestRun?.id ?? null,
|
||
retryReason: "provider_quota_recovery",
|
||
providerQuotaRetryNotBefore: retryAt.toISOString(),
|
||
},
|
||
"normal_model",
|
||
),
|
||
status: "queued",
|
||
requestedByActorType: "system",
|
||
requestedByActorId: null,
|
||
idempotencyKey: `provider_quota_recovery:${input.issue.id}:${retryAt.toISOString()}`,
|
||
updatedAt: now,
|
||
})
|
||
.returning()
|
||
.then((rows) => rows[0]!);
|
||
const scheduledRun = await tx
|
||
.insert(heartbeatRuns)
|
||
.values({
|
||
companyId: input.issue.companyId,
|
||
agentId: input.agentId,
|
||
scopeKind: "issue",
|
||
issueId: input.issue.id,
|
||
invocationSource: "automation",
|
||
triggerDetail: "system",
|
||
status: "scheduled_retry",
|
||
wakeupRequestId: wakeup.id,
|
||
retryOfRunId: input.latestRun?.id ?? null,
|
||
scheduledRetryAt: retryAt,
|
||
scheduledRetryAttempt: 1,
|
||
scheduledRetryReason: "provider_quota_recovery",
|
||
contextSnapshot: withRecoveryContext(
|
||
{
|
||
issueId: input.issue.id,
|
||
taskId: input.issue.id,
|
||
wakeReason: "provider_quota_recovery",
|
||
retryReason: "provider_quota_recovery",
|
||
providerQuotaRetryNotBefore: retryAt.toISOString(),
|
||
},
|
||
"normal_model",
|
||
),
|
||
updatedAt: now,
|
||
})
|
||
.returning()
|
||
.then((rows) => rows[0]!);
|
||
await tx
|
||
.update(agentWakeupRequests)
|
||
.set({ runId: scheduledRun.id, updatedAt: now })
|
||
.where(eq(agentWakeupRequests.id, wakeup.id));
|
||
await tx
|
||
.update(issueRecoveryActions)
|
||
.set({
|
||
monitorPolicy: {
|
||
type: "wait_recovery",
|
||
retryAgentId: input.agentId,
|
||
scheduledRunId: scheduledRun.id,
|
||
retryAt: retryAt.toISOString(),
|
||
},
|
||
timeoutAt: retryAt,
|
||
updatedAt: now,
|
||
})
|
||
.where(eq(issueRecoveryActions.id, input.actionId));
|
||
return scheduledRun;
|
||
});
|
||
}
|
||
|
||
function buildRecoveryIssueInPlaceEscalationComment(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
previousStatus: StrandedPreviousStatus;
|
||
latestRun: LatestIssueRun;
|
||
prefix: string;
|
||
}) {
|
||
const runLink = input.latestRun
|
||
? runUiLink(
|
||
{ id: input.latestRun.id, agentId: input.latestRun.agentId },
|
||
input.prefix,
|
||
)
|
||
: "none";
|
||
const retryReason =
|
||
readNonEmptyString(
|
||
parseObject(input.latestRun?.contextSnapshot)?.retryReason,
|
||
) ?? "none";
|
||
const failureSummary = summarizeRunFailureForIssueComment(input.latestRun);
|
||
|
||
return [
|
||
"Paperclip stopped automatic stranded-work recovery for this recovery issue.",
|
||
"",
|
||
`- Recovery issue: ${issueUiLink({ identifier: input.issue.identifier, id: input.issue.id }, input.prefix)}`,
|
||
`- Previous status: \`${input.previousStatus}\``,
|
||
`- Latest run: ${runLink}`,
|
||
`- Latest run status: \`${input.latestRun?.status ?? "unknown"}\``,
|
||
`- Retry reason: \`${retryReason}\``,
|
||
failureSummary
|
||
? `- Failure: ${failureSummary.trim()}`
|
||
: "- Failure: none recorded",
|
||
"- Guard: recovery issues do not create nested `stranded_issue_recovery` issues.",
|
||
"",
|
||
"Next action: the current recovery owner should inspect the failed run evidence, restore a live execution path or record the manual resolution, then move this recovery issue out of `blocked`.",
|
||
].join("\n");
|
||
}
|
||
|
||
async function escalateStrandedRecoveryIssueInPlace(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
previousStatus: StrandedPreviousStatus;
|
||
latestRun: LatestIssueRun;
|
||
}) {
|
||
const updated = await issuesSvc.update(input.issue.id, {
|
||
status: "blocked",
|
||
});
|
||
if (!updated) return null;
|
||
|
||
const prefix = await getCompanyIssuePrefix(input.issue.companyId);
|
||
await issuesSvc.addComment(
|
||
input.issue.id,
|
||
buildRecoveryIssueInPlaceEscalationComment({
|
||
issue: input.issue,
|
||
previousStatus: input.previousStatus,
|
||
latestRun: input.latestRun,
|
||
prefix,
|
||
}),
|
||
{},
|
||
{
|
||
authorType: "system",
|
||
presentation: compactRecoveryPresentation(
|
||
"Recovery: recovery attempt failed — remains blocked",
|
||
),
|
||
metadata: {
|
||
version: 1,
|
||
sourceRunId: input.latestRun?.id ?? null,
|
||
sections: [
|
||
{
|
||
title: "Recovery",
|
||
rows: [
|
||
{
|
||
type: "key_value",
|
||
label: "Cause",
|
||
value: "recovery_issue_failed",
|
||
},
|
||
{
|
||
type: "key_value",
|
||
label: "Previous status",
|
||
value: input.previousStatus,
|
||
},
|
||
...(input.latestRun
|
||
? [
|
||
{
|
||
type: "run_link" as const,
|
||
label: "Latest run",
|
||
runId: input.latestRun.id,
|
||
title: input.latestRun.status,
|
||
},
|
||
]
|
||
: []),
|
||
],
|
||
},
|
||
],
|
||
},
|
||
},
|
||
);
|
||
|
||
await logActivity(db, {
|
||
companyId: input.issue.companyId,
|
||
actorType: "system",
|
||
actorId: "system",
|
||
agentId: null,
|
||
runId: null,
|
||
action: "issue.updated",
|
||
entityType: "issue",
|
||
entityId: input.issue.id,
|
||
details: {
|
||
identifier: input.issue.identifier,
|
||
status: "blocked",
|
||
previousStatus: input.previousStatus,
|
||
source: "recovery.reconcile_stranded_recovery_issue",
|
||
latestRunId: input.latestRun?.id ?? null,
|
||
latestRunStatus: input.latestRun?.status ?? null,
|
||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||
originKind: input.issue.originKind,
|
||
originId: input.issue.originId,
|
||
},
|
||
});
|
||
|
||
return updated;
|
||
}
|
||
|
||
async function existingBlockerIssueIds(companyId: string, issueId: string) {
|
||
return db
|
||
.select({ blockerIssueId: issueRelations.issueId })
|
||
.from(issueRelations)
|
||
.where(
|
||
and(
|
||
eq(issueRelations.companyId, companyId),
|
||
eq(issueRelations.relatedIssueId, issueId),
|
||
eq(issueRelations.type, "blocks"),
|
||
),
|
||
)
|
||
.then((rows) => rows.map((row) => row.blockerIssueId));
|
||
}
|
||
|
||
async function existingUnresolvedBlockerIssues(
|
||
companyId: string,
|
||
issueId: string,
|
||
) {
|
||
return db
|
||
.select({ id: issueRelations.issueId, identifier: issues.identifier })
|
||
.from(issueRelations)
|
||
.innerJoin(
|
||
issues,
|
||
and(
|
||
eq(issues.companyId, issueRelations.companyId),
|
||
eq(issues.id, issueRelations.issueId),
|
||
),
|
||
)
|
||
.where(
|
||
and(
|
||
eq(issueRelations.companyId, companyId),
|
||
eq(issueRelations.relatedIssueId, issueId),
|
||
eq(issueRelations.type, "blocks"),
|
||
notInArray(issues.status, ["done", "cancelled"]),
|
||
),
|
||
);
|
||
}
|
||
|
||
async function existingUnresolvedBlockerIssueIds(
|
||
companyId: string,
|
||
issueId: string,
|
||
) {
|
||
return existingUnresolvedBlockerIssues(companyId, issueId).then((rows) =>
|
||
rows.map((row) => row.id),
|
||
);
|
||
}
|
||
|
||
async function openChildIssues(issue: typeof issues.$inferSelect) {
|
||
return db
|
||
.select({ id: issues.id, identifier: issues.identifier })
|
||
.from(issues)
|
||
.where(
|
||
and(
|
||
eq(issues.companyId, issue.companyId),
|
||
eq(issues.parentId, issue.id),
|
||
visibleIssueCondition(),
|
||
notInArray(issues.status, ["done", "cancelled"]),
|
||
),
|
||
);
|
||
}
|
||
|
||
async function healthyOpenChildIssues(issue: typeof issues.$inferSelect, sameWorkspaceOnly = false) {
|
||
if (sameWorkspaceOnly && !issue.projectWorkspaceId) return [];
|
||
const childCandidates = await db
|
||
.select()
|
||
.from(issues)
|
||
.where(
|
||
and(
|
||
eq(issues.companyId, issue.companyId),
|
||
eq(issues.parentId, issue.id),
|
||
...(sameWorkspaceOnly ? [eq(issues.projectWorkspaceId, issue.projectWorkspaceId!)] : []),
|
||
visibleIssueCondition(),
|
||
notInArray(issues.status, ["done", "cancelled"]),
|
||
),
|
||
);
|
||
const openChildren = [] as Array<{ id: string; identifier: string | null }>;
|
||
for (const child of childCandidates) {
|
||
const childState = await collectDispositionRepairSourceState(db, {
|
||
issue: child,
|
||
});
|
||
if (
|
||
childState.hasActiveExecutionPath ||
|
||
(!sameWorkspaceOnly && childState.hasDurableWaitingPath)
|
||
) {
|
||
openChildren.push({ id: child.id, identifier: child.identifier });
|
||
}
|
||
}
|
||
return openChildren;
|
||
}
|
||
|
||
async function resolveContinuationWaitingOnReview(
|
||
issue: typeof issues.$inferSelect,
|
||
) {
|
||
const [existingBlockers, openChildren] = await Promise.all([
|
||
existingUnresolvedBlockerIssues(issue.companyId, issue.id),
|
||
openChildIssues(issue),
|
||
]);
|
||
const blockedByIssueIds = [
|
||
...new Set([
|
||
...existingBlockers.map((row) => row.id),
|
||
...openChildren.map((row) => row.id),
|
||
]),
|
||
];
|
||
if (blockedByIssueIds.length === 0) return null;
|
||
|
||
const updated = await issuesSvc.update(issue.id, {
|
||
status: "blocked",
|
||
blockedByIssueIds,
|
||
});
|
||
if (!updated) return null;
|
||
|
||
const waitingOn = formatIssueLinksForComment([
|
||
...openChildren,
|
||
...existingBlockers,
|
||
]);
|
||
await issuesSvc.addComment(
|
||
issue.id,
|
||
`This task is waiting on ${waitingOn} to finish. ` +
|
||
"It will continue automatically when that work is done — there's nothing you need to do. " +
|
||
"(It was paused because the latest run reported it was waiting for review/approval; " +
|
||
"Paperclip turned that into a normal dependency wait instead of flagging it as stuck.)",
|
||
{},
|
||
{
|
||
authorType: "system",
|
||
presentation: compactRecoveryPresentation(
|
||
"Recovery: waiting on dependencies — moved to blocked",
|
||
),
|
||
metadata: {
|
||
version: 1,
|
||
sections: [
|
||
{
|
||
title: "Recovery",
|
||
rows: [
|
||
{
|
||
type: "key_value",
|
||
label: "Cause",
|
||
value: "continuation_waiting_on_review",
|
||
},
|
||
{
|
||
type: "key_value",
|
||
label: "Previous status",
|
||
value: issue.status,
|
||
},
|
||
{
|
||
type: "key_value",
|
||
label: "Blocking issues",
|
||
value: blockedByIssueIds.join(", ").slice(0, 2000),
|
||
},
|
||
],
|
||
},
|
||
],
|
||
},
|
||
},
|
||
);
|
||
await logActivity(db, {
|
||
companyId: issue.companyId,
|
||
actorType: "system",
|
||
actorId: "system",
|
||
agentId: null,
|
||
runId: null,
|
||
action: "issue.updated",
|
||
entityType: "issue",
|
||
entityId: issue.id,
|
||
details: {
|
||
identifier: issue.identifier,
|
||
status: "blocked",
|
||
previousStatus: issue.status,
|
||
source: "recovery.reconcile_continuation_waiting_on_review",
|
||
blockedByIssueIds,
|
||
},
|
||
});
|
||
return updated;
|
||
}
|
||
|
||
function readDispositionRepairAttempt(latestRun: LatestIssueRun) {
|
||
if (!latestRun) return null;
|
||
const context = parseObject(latestRun.contextSnapshot);
|
||
if (
|
||
readNonEmptyString(context.retryReason) !==
|
||
ISSUE_DISPOSITION_REPAIR_RETRY_REASON
|
||
)
|
||
return null;
|
||
return {
|
||
attempt: Math.max(
|
||
1,
|
||
Math.floor(asNumber(context.dispositionRepairAttempt, 1)),
|
||
),
|
||
fingerprint: readNonEmptyString(context.dispositionRepairFingerprint),
|
||
};
|
||
}
|
||
|
||
async function resolveDispositionRepairActionAsCovered(
|
||
issue: typeof issues.$inferSelect,
|
||
reason: string,
|
||
) {
|
||
const active = await recoveryActionsSvc.getActiveForIssue(
|
||
issue.companyId,
|
||
issue.id,
|
||
);
|
||
if (!active || active.kind !== "deliberate_wait_without_target") return;
|
||
await recoveryActionsSvc.resolveActiveForIssue({
|
||
companyId: issue.companyId,
|
||
sourceIssueId: issue.id,
|
||
actionId: active.id,
|
||
status: "resolved",
|
||
outcome: "restored",
|
||
resolutionNote: reason,
|
||
});
|
||
await logActivity(db, {
|
||
companyId: issue.companyId,
|
||
actorType: "system",
|
||
actorId: "recovery",
|
||
agentId: null,
|
||
runId: null,
|
||
action: "issue.disposition_repair_resolved",
|
||
entityType: "issue_recovery_action",
|
||
entityId: active.id,
|
||
details: {
|
||
sourceIssueId: issue.id,
|
||
sourceIdentifier: issue.identifier,
|
||
reason,
|
||
},
|
||
});
|
||
}
|
||
|
||
async function ensureDispositionRepairAction(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
latestRun: LatestIssueRun;
|
||
fingerprint: string;
|
||
attemptCount: number;
|
||
legacyEpisode?: LegacyDispositionEpisode;
|
||
}) {
|
||
let active = await recoveryActionsSvc.getActiveForIssue(
|
||
input.issue.companyId,
|
||
input.issue.id,
|
||
);
|
||
if (
|
||
active &&
|
||
(active.kind !== "deliberate_wait_without_target" ||
|
||
active.fingerprint !== input.fingerprint)
|
||
) {
|
||
await recoveryActionsSvc.resolveActiveForIssue({
|
||
companyId: input.issue.companyId,
|
||
sourceIssueId: input.issue.id,
|
||
actionId: active.id,
|
||
status: "cancelled",
|
||
outcome: "cancelled",
|
||
resolutionNote: "source_state_changed",
|
||
});
|
||
await logActivity(db, {
|
||
companyId: input.issue.companyId,
|
||
actorType: "system",
|
||
actorId: "recovery",
|
||
agentId: null,
|
||
runId: input.latestRun?.id ?? null,
|
||
action: "issue.disposition_repair_fingerprint_reset",
|
||
entityType: "issue_recovery_action",
|
||
entityId: active.id,
|
||
details: {
|
||
sourceIssueId: input.issue.id,
|
||
previousFingerprint: active.fingerprint,
|
||
nextFingerprint: input.fingerprint,
|
||
terminalReason: "source_state_changed",
|
||
},
|
||
});
|
||
active = null;
|
||
}
|
||
|
||
if (active && active.attemptCount >= input.attemptCount) return active;
|
||
|
||
return recoveryActionsSvc.upsertSourceScoped({
|
||
companyId: input.issue.companyId,
|
||
sourceIssueId: input.issue.id,
|
||
kind: "deliberate_wait_without_target",
|
||
ownerType: "agent",
|
||
ownerAgentId: input.issue.assigneeAgentId,
|
||
previousOwnerAgentId: input.issue.assigneeAgentId,
|
||
returnOwnerAgentId: input.issue.assigneeAgentId,
|
||
cause: "deliberate_wait_without_target",
|
||
fingerprint: input.fingerprint,
|
||
evidence: {
|
||
sourceIssueId: input.issue.id,
|
||
sourceIdentifier: input.issue.identifier,
|
||
latestRunId: input.latestRun?.id ?? null,
|
||
latestRunStatus: input.latestRun?.status ?? null,
|
||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||
sourceStateFingerprint: input.fingerprint,
|
||
terminalReason: null,
|
||
},
|
||
nextAction:
|
||
"The original owner must replace the parked summary with a terminal, live, blocked, monitored, or typed waiting disposition.",
|
||
wakePolicy: {
|
||
type: "bounded_owner_disposition_repair",
|
||
retryAgentId: input.issue.assigneeAgentId,
|
||
attempt: input.attemptCount,
|
||
maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
},
|
||
maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
attemptCount: input.attemptCount,
|
||
lastAttemptAt: new Date(),
|
||
});
|
||
}
|
||
|
||
async function scheduleDispositionRepairAttempt(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
latestRun: LatestIssueRun;
|
||
action: Awaited<ReturnType<typeof ensureDispositionRepairAction>>;
|
||
fingerprint: string;
|
||
attempt: number;
|
||
legacyEpisode?: LegacyDispositionEpisode;
|
||
}) {
|
||
const agentId = input.issue.assigneeAgentId;
|
||
if (!agentId) return null;
|
||
// Preserve the initiating identity on the durable row, including while a
|
||
// delayed repair is waiting to dispatch. Never substitute the issue owner.
|
||
const sourceRun = input.latestRun?.id ? await db.select()
|
||
.from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.latestRun.id), eq(heartbeatRuns.companyId, input.issue.companyId)))
|
||
.limit(1).then(rows => rows[0]) : null;
|
||
const timing = dispositionRepairDelayMs(input.attempt, input.fingerprint);
|
||
const now = new Date();
|
||
const retryAt = new Date(now.getTime() + timing.delayMs);
|
||
const idempotencyKey = `issue_disposition_repair:${input.issue.id}:${input.fingerprint}:${input.attempt}`;
|
||
const context = withRecoveryContext(
|
||
{
|
||
issueId: input.issue.id,
|
||
taskId: input.issue.id,
|
||
wakeReason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||
retryReason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||
source: "issue.deliberate_wait_disposition_repair",
|
||
executionRetryAccounting: executionRetryAccounting(sourceRun ?? {}),
|
||
retryOfRunId: input.latestRun?.id ?? null,
|
||
recoveryActionId: input.action.id,
|
||
dispositionRepairFingerprint: input.fingerprint,
|
||
dispositionRepairAttempt: input.attempt,
|
||
dispositionRepairMaxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
...(input.legacyEpisode ? {
|
||
legacyDispositionEpisode: { ...input.legacyEpisode, attempt: input.attempt },
|
||
dispositionRepairSourceRunId: input.latestRun?.id ?? null,
|
||
} : {}),
|
||
bypassContinuationSummaryPark: true,
|
||
dispositionRepairInstruction:
|
||
input.legacyEpisode ? LEGACY_DISPOSITION_REPAIR_INSTRUCTION : "Revalidate the issue and replace the invalid parked summary with a durable disposition. Continue productive work when appropriate.",
|
||
},
|
||
"normal_model",
|
||
);
|
||
|
||
const findScheduledRun = () =>
|
||
db
|
||
.select({ run: heartbeatRuns })
|
||
.from(agentWakeupRequests)
|
||
.innerJoin(
|
||
heartbeatRuns,
|
||
eq(heartbeatRuns.id, agentWakeupRequests.runId),
|
||
)
|
||
.where(
|
||
and(
|
||
eq(agentWakeupRequests.companyId, input.issue.companyId),
|
||
eq(agentWakeupRequests.idempotencyKey, idempotencyKey),
|
||
sql`${agentWakeupRequests.status} <> 'skipped'`,
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0]?.run ?? null);
|
||
|
||
let scheduledRun = await findScheduledRun();
|
||
let created = false;
|
||
if (!scheduledRun) {
|
||
try {
|
||
if (timing.delayMs === 0) {
|
||
const enqueuedRun = await deps.enqueueWakeup(agentId, {
|
||
source: "automation",
|
||
triggerDetail: "system",
|
||
reason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||
idempotencyKey,
|
||
payload: withRecoveryContext(
|
||
{
|
||
issueId: input.issue.id,
|
||
retryOfRunId: input.latestRun?.id ?? null,
|
||
recoveryActionId: input.action.id,
|
||
dispositionRepairFingerprint: input.fingerprint,
|
||
dispositionRepairAttempt: input.attempt,
|
||
bypassContinuationSummaryPark: true,
|
||
},
|
||
"normal_model",
|
||
),
|
||
requestedByActorType: "system",
|
||
requestedByActorId: null,
|
||
contextSnapshot: context,
|
||
...(input.legacyEpisode ? { issueStateGuard: { statuses: [input.issue.status], assigneeAgentId: agentId } } : {}),
|
||
});
|
||
scheduledRun = enqueuedRun ?? (await findScheduledRun());
|
||
created = Boolean(enqueuedRun);
|
||
} else {
|
||
scheduledRun = await db.transaction(async (tx) => {
|
||
const wakeup = await tx
|
||
.insert(agentWakeupRequests)
|
||
.values({
|
||
companyId: input.issue.companyId,
|
||
agentId,
|
||
source: "automation",
|
||
triggerDetail: "system",
|
||
reason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||
payload: withRecoveryContext(
|
||
{
|
||
issueId: input.issue.id,
|
||
retryOfRunId: input.latestRun?.id ?? null,
|
||
recoveryActionId: input.action.id,
|
||
dispositionRepairFingerprint: input.fingerprint,
|
||
dispositionRepairAttempt: input.attempt,
|
||
bypassContinuationSummaryPark: true,
|
||
},
|
||
"normal_model",
|
||
),
|
||
status: "queued",
|
||
requestedByActorType: "system",
|
||
requestedByActorId: null,
|
||
idempotencyKey,
|
||
updatedAt: now,
|
||
})
|
||
.returning()
|
||
.then((rows) => rows[0]!);
|
||
const run = await tx
|
||
.insert(heartbeatRuns)
|
||
.values({
|
||
companyId: input.issue.companyId,
|
||
agentId,
|
||
invocationSource: "automation",
|
||
triggerDetail: "system",
|
||
status: "scheduled_retry",
|
||
wakeupRequestId: wakeup.id,
|
||
retryOfRunId: input.latestRun?.id ?? null,
|
||
scheduledRetryAt: retryAt,
|
||
scheduledRetryAttempt: input.attempt,
|
||
scheduledRetryReason: ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||
responsibleUserId: sourceRun?.responsibleUserId ?? null,
|
||
contextSnapshot: context,
|
||
updatedAt: now,
|
||
})
|
||
.returning()
|
||
.then((rows) => rows[0]!);
|
||
await tx
|
||
.update(agentWakeupRequests)
|
||
.set({ runId: run.id, updatedAt: now })
|
||
.where(eq(agentWakeupRequests.id, wakeup.id));
|
||
return run;
|
||
});
|
||
created = true;
|
||
}
|
||
} catch (error) {
|
||
if (!isUniqueViolation(error, DISPOSITION_REPAIR_IDEMPOTENCY_INDEX))
|
||
throw error;
|
||
const winningRun = await findScheduledRun();
|
||
if (!winningRun) throw error;
|
||
scheduledRun = winningRun;
|
||
}
|
||
}
|
||
|
||
if (!scheduledRun) return null;
|
||
|
||
await db
|
||
.update(issueRecoveryActions)
|
||
.set({
|
||
attemptCount: input.attempt,
|
||
wakePolicy: {
|
||
type: "bounded_owner_disposition_repair",
|
||
retryAgentId: agentId,
|
||
attempt: input.attempt,
|
||
maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
baseBackoffMs: timing.baseDelayMs,
|
||
jitterMs: timing.jitterMs,
|
||
retryAt: retryAt.toISOString(),
|
||
scheduledRunId: scheduledRun.id,
|
||
},
|
||
timeoutAt: retryAt,
|
||
lastAttemptAt: now,
|
||
updatedAt: now,
|
||
})
|
||
.where(
|
||
and(
|
||
eq(issueRecoveryActions.id, input.action.id),
|
||
eq(issueRecoveryActions.companyId, input.issue.companyId),
|
||
// A fast successor can reserve the next slot before enqueue returns.
|
||
// An older scheduler must not rewind its ledger or resurrect a wait.
|
||
eq(issueRecoveryActions.status, "active"),
|
||
eq(issueRecoveryActions.ownerType, "agent"),
|
||
eq(issueRecoveryActions.fingerprint, input.fingerprint),
|
||
sql`${issueRecoveryActions.attemptCount} <= ${input.attempt}`,
|
||
),
|
||
);
|
||
|
||
if (created) {
|
||
await logActivity(db, {
|
||
companyId: input.issue.companyId,
|
||
actorType: "system",
|
||
actorId: "recovery",
|
||
agentId: null,
|
||
runId: input.latestRun?.id ?? null,
|
||
action: "issue.disposition_repair_scheduled",
|
||
entityType: "issue_recovery_action",
|
||
entityId: input.action.id,
|
||
details: {
|
||
sourceIssueId: input.issue.id,
|
||
sourceIdentifier: input.issue.identifier,
|
||
ownerAgentId: agentId,
|
||
sourceStateFingerprint: input.fingerprint,
|
||
attempt: input.attempt,
|
||
maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
baseBackoffMs: timing.baseDelayMs,
|
||
jitterMs: timing.jitterMs,
|
||
retryAt: retryAt.toISOString(),
|
||
scheduledRunId: scheduledRun.id,
|
||
},
|
||
});
|
||
}
|
||
|
||
return scheduledRun;
|
||
}
|
||
|
||
async function latestRecoveryActionRun(
|
||
action: typeof issueRecoveryActions.$inferSelect,
|
||
) {
|
||
return db
|
||
.select()
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, action.companyId),
|
||
sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId' = ${action.id}`,
|
||
),
|
||
)
|
||
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id))
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
}
|
||
|
||
async function sourceHasNewPathOutsideRecoveryAction(
|
||
action: typeof issueRecoveryActions.$inferSelect,
|
||
) {
|
||
const [run, wake] = await Promise.all([
|
||
db
|
||
.select({ id: heartbeatRuns.id })
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, action.companyId),
|
||
inArray(heartbeatRuns.status, [
|
||
...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,
|
||
]),
|
||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId') = ${action.sourceIssueId}`,
|
||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId', '') <> ${action.id}`,
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null),
|
||
db
|
||
.select({ id: agentWakeupRequests.id })
|
||
.from(agentWakeupRequests)
|
||
.where(
|
||
and(
|
||
eq(agentWakeupRequests.companyId, action.companyId),
|
||
inArray(agentWakeupRequests.status, [
|
||
"queued",
|
||
"claimed",
|
||
"deferred_issue_execution",
|
||
]),
|
||
sql`coalesce(${agentWakeupRequests.payload} ->> 'issueId', ${agentWakeupRequests.payload} ->> 'taskId') = ${action.sourceIssueId}`,
|
||
sql`coalesce(${agentWakeupRequests.payload} ->> 'recoveryActionId', '') <> ${action.id}`,
|
||
),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null),
|
||
]);
|
||
return Boolean(run || wake);
|
||
}
|
||
|
||
async function reconcileActiveRecoveryActions() {
|
||
const rows = await db
|
||
.select({ action: issueRecoveryActions, issue: issues })
|
||
.from(issueRecoveryActions)
|
||
.innerJoin(
|
||
issues,
|
||
and(
|
||
eq(issues.id, issueRecoveryActions.sourceIssueId),
|
||
eq(issues.companyId, issueRecoveryActions.companyId),
|
||
),
|
||
)
|
||
.where(inArray(issueRecoveryActions.status, ["active", "escalated"]));
|
||
|
||
const result = {
|
||
requeued: 0,
|
||
escalated: 0,
|
||
resolved: 0,
|
||
skipped: 0,
|
||
issueIds: [] as string[],
|
||
};
|
||
for (const { action, issue } of rows) {
|
||
if (await hasUnverifiedNativeWorkspaceFinalizationOwner(db, action)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
const wakePolicy = parseObject(action.wakePolicy);
|
||
const wakePolicyType = readNonEmptyString(wakePolicy.type);
|
||
if (
|
||
wakePolicyType !== "bounded_recovery_owner" &&
|
||
wakePolicyType !== "bounded_owner_disposition_repair" &&
|
||
action.ownerType !== "board"
|
||
) {
|
||
continue;
|
||
}
|
||
|
||
if (issue.status === "done" || issue.status === "cancelled") {
|
||
const resolved = await recoveryActionsSvc.resolveActiveForIssue({
|
||
companyId: action.companyId,
|
||
sourceIssueId: action.sourceIssueId,
|
||
actionId: action.id,
|
||
status: "resolved",
|
||
outcome: "restored",
|
||
resolutionNote: "source_terminal",
|
||
});
|
||
if (resolved) {
|
||
result.resolved += 1;
|
||
result.issueIds.push(issue.id);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
// A queued comment or healthy child cannot establish what the stopped
|
||
// provider already did. Only execution reconciliation can clear this hold.
|
||
if (requiresExecutionReconciliation(action.cause)
|
||
|| isNativeWorkspaceExportRepairCause(action.cause)
|
||
|| action.cause === "native_workspace_finalization_owner_unverified"
|
||
|| action.cause === "native_workspace_sync_out_unsafe_archive") {
|
||
// A queued wake or healthy child does not export this accepted result.
|
||
// Only its native finalizer or an explicit board disposition can settle it.
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const [sourceState, healthyChildren, hasNewSourcePath] =
|
||
await Promise.all([
|
||
collectDispositionRepairSourceState(db, { issue }),
|
||
healthyOpenChildIssues(issue),
|
||
sourceHasNewPathOutsideRecoveryAction(action),
|
||
]);
|
||
const durablePathRestored =
|
||
action.ownerType !== "board" && sourceState.hasDurableWaitingPath;
|
||
if (
|
||
durablePathRestored ||
|
||
healthyChildren.length > 0 ||
|
||
hasNewSourcePath
|
||
) {
|
||
if (healthyChildren.length > 0 && !sourceState.hasDurableWaitingPath) {
|
||
const blockerIds = await existingUnresolvedBlockerIssueIds(
|
||
issue.companyId,
|
||
issue.id,
|
||
);
|
||
await issuesSvc.update(issue.id, {
|
||
status: "blocked",
|
||
blockedByIssueIds: [
|
||
...new Set([
|
||
...blockerIds,
|
||
...healthyChildren.map((child) => child.id),
|
||
]),
|
||
],
|
||
});
|
||
}
|
||
const resolved = await recoveryActionsSvc.resolveActiveForIssue({
|
||
companyId: action.companyId,
|
||
sourceIssueId: action.sourceIssueId,
|
||
actionId: action.id,
|
||
status: "resolved",
|
||
outcome: "restored",
|
||
resolutionNote: durablePathRestored
|
||
? `durable_path_restored:${sourceState.durablePathReason ?? "unknown"}`
|
||
: healthyChildren.length > 0
|
||
? "durable_path_restored:healthy_child"
|
||
: "new_source_execution_path",
|
||
});
|
||
if (resolved) {
|
||
result.resolved += 1;
|
||
result.issueIds.push(issue.id);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (wakePolicyType === "bounded_owner_disposition_repair") {
|
||
if (
|
||
await isAutomaticRecoverySuppressedByPauseHold(
|
||
db,
|
||
issue.companyId,
|
||
issue.id,
|
||
treeControlSvc,
|
||
)
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const latestRun = await latestRecoveryActionRun(action);
|
||
const persistedAttempt = Math.max(
|
||
action.attemptCount,
|
||
Math.max(
|
||
0,
|
||
Math.floor(asNumber(wakePolicy.attempt, action.attemptCount)),
|
||
),
|
||
);
|
||
const outcome = await reconcileDispositionRepair(issue, latestRun, {
|
||
historicalAttemptCount: persistedAttempt,
|
||
});
|
||
if (outcome === "queued") {
|
||
result.requeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else if (outcome === "escalated") {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (action.ownerType === "board") continue;
|
||
|
||
// Legacy takeover actions remain readable and resolvable, but recovery no
|
||
// longer schedules another agent-owned wake for them.
|
||
result.skipped += 1;
|
||
}
|
||
return result;
|
||
}
|
||
|
||
async function escalateDispositionRepair(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
latestRun: LatestIssueRun;
|
||
fingerprint: string;
|
||
attemptCount: number;
|
||
terminalReason: string;
|
||
legacyEpisode?: LegacyDispositionEpisode;
|
||
}) {
|
||
const action = await ensureDispositionRepairAction({
|
||
issue: input.issue,
|
||
latestRun: input.latestRun,
|
||
fingerprint: input.fingerprint,
|
||
attemptCount: input.attemptCount,
|
||
legacyEpisode: input.legacyEpisode,
|
||
});
|
||
const now = new Date();
|
||
await db
|
||
.update(issueRecoveryActions)
|
||
.set({
|
||
status: "active",
|
||
ownerType: "board",
|
||
ownerAgentId: null,
|
||
ownerUserId: null,
|
||
maxAttempts: null,
|
||
evidence: {
|
||
...action.evidence,
|
||
latestRunId: input.latestRun?.id ?? null,
|
||
latestRunStatus: input.latestRun?.status ?? null,
|
||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||
terminalReason: input.terminalReason,
|
||
sourceAttemptCount: input.attemptCount,
|
||
sourceMaxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
routingPolicy: STRANDED_BOARD_ESCALATION_POLICY,
|
||
},
|
||
nextAction:
|
||
"Inspect the evidence and choose whether to repair, retry the original owner, explicitly reassign, or resolve the source issue.",
|
||
wakePolicy: {
|
||
type: "board_escalation",
|
||
reason: input.terminalReason,
|
||
preservesSourceAssignee: true,
|
||
},
|
||
timeoutAt: null,
|
||
resolutionNote: input.terminalReason,
|
||
updatedAt: now,
|
||
})
|
||
.where(
|
||
and(
|
||
eq(issueRecoveryActions.id, action.id),
|
||
eq(issueRecoveryActions.companyId, input.issue.companyId),
|
||
),
|
||
);
|
||
|
||
const updated = await issuesSvc.update(input.issue.id, {
|
||
status: "blocked",
|
||
});
|
||
if (!updated) return null;
|
||
const sourceAssigneePreserved =
|
||
updated.assigneeAgentId === input.issue.assigneeAgentId &&
|
||
updated.assigneeUserId === input.issue.assigneeUserId;
|
||
|
||
await issuesSvc.addComment(
|
||
input.issue.id,
|
||
[
|
||
"Paperclip exhausted the bounded original-owner disposition repair without a durable source-state change.",
|
||
"",
|
||
`- Attempts: ${input.attemptCount}/${input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS}`,
|
||
`- Terminal reason: \`${input.terminalReason}\``,
|
||
"- Recovery owner: board",
|
||
"- Source ownership: unchanged; reassignment requires an explicit decision or a policy-defined serious failure.",
|
||
"",
|
||
"Next action: repair the liveness disposition or request an explicit source-owner decision.",
|
||
].join("\n"),
|
||
{},
|
||
{
|
||
authorType: "system",
|
||
presentation: compactRecoveryPresentation(
|
||
"Agent needs attention",
|
||
),
|
||
metadata: {
|
||
...recoveryNoticeMetadata({
|
||
cause: "deliberate_wait_without_target",
|
||
latestRun: input.latestRun,
|
||
recoveryActionId: action.id,
|
||
previousStatus: input.issue.status,
|
||
recoveryOwner: null,
|
||
}),
|
||
recovery: {
|
||
kind: "disposition_repair_escalated",
|
||
actionId: action.id,
|
||
attemptCount: input.attemptCount,
|
||
maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
reason: input.terminalReason,
|
||
assigneeAgentId: input.issue.assigneeAgentId,
|
||
},
|
||
},
|
||
},
|
||
);
|
||
|
||
await logActivity(db, {
|
||
companyId: input.issue.companyId,
|
||
actorType: "system",
|
||
actorId: "recovery",
|
||
agentId: null,
|
||
runId: input.latestRun?.id ?? null,
|
||
action: "issue.disposition_repair_escalated",
|
||
entityType: "issue",
|
||
entityId: input.issue.id,
|
||
details: {
|
||
identifier: input.issue.identifier,
|
||
status: "blocked",
|
||
previousStatus: input.issue.status,
|
||
sourceStateFingerprint: input.fingerprint,
|
||
attemptCount: input.attemptCount,
|
||
maxAttempts: input.legacyEpisode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS,
|
||
terminalReason: input.terminalReason,
|
||
recoveryActionId: action.id,
|
||
recoveryOwnerAgentId: null,
|
||
recoveryOwnerType: "board",
|
||
routingPolicy: STRANDED_BOARD_ESCALATION_POLICY,
|
||
sourceAssigneeBefore: {
|
||
agentId: input.issue.assigneeAgentId,
|
||
userId: input.issue.assigneeUserId,
|
||
},
|
||
sourceAssigneeAfter: {
|
||
agentId: updated.assigneeAgentId,
|
||
userId: updated.assigneeUserId,
|
||
},
|
||
sourceAssigneePreserved,
|
||
},
|
||
});
|
||
if (!sourceAssigneePreserved) {
|
||
logger.error(
|
||
{
|
||
issueId: input.issue.id,
|
||
beforeAssigneeAgentId: input.issue.assigneeAgentId,
|
||
afterAssigneeAgentId: updated.assigneeAgentId,
|
||
beforeAssigneeUserId: input.issue.assigneeUserId,
|
||
afterAssigneeUserId: updated.assigneeUserId,
|
||
},
|
||
"automatic disposition recovery observed a concurrent source-owner change",
|
||
);
|
||
}
|
||
return updated;
|
||
}
|
||
|
||
async function decidePersistedLegacyContinuation(
|
||
issue: typeof issues.$inferSelect,
|
||
runId: string,
|
||
state: Awaited<ReturnType<typeof collectDispositionRepairSourceState>>,
|
||
episode: LegacyDispositionEpisode,
|
||
dispatchRunId?: string,
|
||
) {
|
||
const [run] = await db.select().from(heartbeatRuns).where(and(
|
||
eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, issue.companyId),
|
||
)).limit(1);
|
||
if (!run) return { kind: "skip" as const, reason: "missing_run" };
|
||
const context = parseObject(run.contextSnapshot);
|
||
if ((readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId)) !== issue.id) {
|
||
return { kind: "skip" as const, reason: "invalid_issue_binding" };
|
||
}
|
||
const [agent, pause, budget, stop, active, routine, goal, latest, durableWait, workspaceChildren] = await Promise.all([
|
||
getAgent(run.agentId),
|
||
isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc),
|
||
isInvocationBudgetBlocked(issue, run.agentId),
|
||
readChatControlRecoveryStop(db, { companyId: issue.companyId, issueId: issue.id, agentId: run.agentId, sourceRunId: run.id }),
|
||
recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id),
|
||
db.select({ id: routines.id }).from(routines).where(and(eq(routines.companyId, issue.companyId), eq(routines.parentIssueId, issue.id), eq(routines.status, "active"))).limit(1),
|
||
db.select({ status: agentTaskSessions.goalStatus }).from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, issue.companyId), eq(agentTaskSessions.agentId, run.agentId), eq(agentTaskSessions.taskKey, issue.id))).limit(1),
|
||
getLatestIssueRun(issue.companyId, issue.id),
|
||
hasPersistedDurableWaitPath(issue, run),
|
||
parseObject(context.paperclipWorkspace).mode === "shared_workspace" ? healthyOpenChildIssues(issue, true) : Promise.resolve([]),
|
||
]);
|
||
const ownsRepair = active?.kind === "deliberate_wait_without_target" && active.ownerType === "agent";
|
||
return decideLegacyContinuation({
|
||
run, issue, agent, episode,
|
||
gates: {
|
||
stopped: stop.kind !== "clear" || isOperatorCancelledRun(run, run.agentId),
|
||
paused: pause, budgetBlocked: budget,
|
||
pendingWait: state.hasDurableWaitingPath || durableWait || parseIssueExecutionState(issue.executionState)?.status === "pending" || Boolean(issue.monitorNextCheckAt),
|
||
activeExecution: state.hasActiveExecutionPath || latest?.id !== (dispatchRunId ?? run.id),
|
||
ownedLifecycle: run.issueCommentStatus === "retry_queued" || run.issueCommentStatus === "retry_exhausted" ||
|
||
Boolean(readNonEmptyString(context.goalControlRequestId)) || context.resumeSessionGoalHeartbeat === true ||
|
||
isPluginManagedIssueLifecycle(issue) || routine.length > 0 || workspaceChildren.length > 0 ||
|
||
Boolean(goal[0]?.status && goal[0].status !== "complete") || Boolean(active && !ownsRepair),
|
||
conversation: Boolean(issue.conversationAgentId) || isWaitingConversation(issue),
|
||
agentInvokable: Boolean(agent && await isAgentInvokable(agent) && isHeartbeatWakeOnDemandEnabled(agent)),
|
||
},
|
||
});
|
||
}
|
||
|
||
async function legacyRepairDispatchBlock(runId: string): Promise<string | null> {
|
||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).limit(1);
|
||
const context = parseObject(run?.contextSnapshot);
|
||
if (!run || !parseObject(context.legacyDispositionEpisode).id) return null;
|
||
if (!["queued", "running", "scheduled_retry"].includes(run.status)) return "repair_not_active";
|
||
const episode = legacyDispositionEpisode(run);
|
||
// Infrastructure retries retain the successful disposition source even
|
||
// though their immediate retryOfRunId points at a failed repair attempt.
|
||
const sourceId = readNonEmptyString(context.dispositionRepairSourceRunId) ?? readNonEmptyString(context.retryOfRunId);
|
||
const issueId = readNonEmptyString(context.issueId);
|
||
if (!sourceId || !issueId || episode.attempt < 1 || episode.attempt > episode.maxAttempts) return "invalid_repair_binding";
|
||
const [source] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, sourceId), eq(heartbeatRuns.companyId, run.companyId))).limit(1);
|
||
if (!source || source.agentId !== run.agentId || legacyDispositionEpisode(source).id !== episode.id) return "invalid_repair_source";
|
||
const [issue] = await db.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))).limit(1);
|
||
if (!issue) return "missing_issue";
|
||
const state = await collectDispositionRepairSourceState(db, { issue, excludeRunId: run.id, excludeWakeupRequestId: run.wakeupRequestId ?? undefined });
|
||
// This slot has already been reserved. Check admission for that slot rather
|
||
// than allocating or charging another repair attempt during dispatch.
|
||
const decision = await decidePersistedLegacyContinuation(issue, source.id, state, { ...episode, attempt: episode.attempt - 1 }, run.id);
|
||
return decision.kind === "enqueue" ? null : decision.kind === "skip" ? decision.reason : "repair_exhausted";
|
||
}
|
||
|
||
async function reconcileLegacyContinuation(runId: string) {
|
||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).limit(1);
|
||
if (!run || run.runtimeMode === "native" || run.status !== "succeeded") return "skipped" as const;
|
||
const context = parseObject(run.contextSnapshot);
|
||
const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId);
|
||
if (!issueId) return "skipped" as const;
|
||
const [issue] = await db.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))).limit(1);
|
||
if (!issue) return "skipped" as const;
|
||
return reconcileDispositionRepair(issue, run, { legacyEpisode: legacyDispositionEpisode(run) });
|
||
}
|
||
|
||
async function reconcileDispositionRepair(
|
||
issue: typeof issues.$inferSelect,
|
||
latestRun: LatestIssueRun,
|
||
options: { historicalAttemptCount?: number; legacyEpisode?: LegacyDispositionEpisode } = {},
|
||
): Promise<"queued" | "escalated" | "covered" | "skipped"> {
|
||
const current = await db
|
||
.select()
|
||
.from(issues)
|
||
.where(
|
||
and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id)),
|
||
)
|
||
.limit(1)
|
||
.then((rows) => rows[0] ?? null);
|
||
if (!current || current.status === "done" || current.status === "cancelled")
|
||
return "skipped";
|
||
|
||
const episode = options.legacyEpisode ?? (
|
||
parseObject(parseObject(latestRun?.contextSnapshot).legacyDispositionEpisode).id && latestRun
|
||
? legacyDispositionEpisode(latestRun) : undefined
|
||
);
|
||
const maxAttempts = episode?.maxAttempts ?? DISPOSITION_REPAIR_MAX_ATTEMPTS;
|
||
const dependencyWait = episode ? null : await resolveContinuationWaitingOnReview(current);
|
||
if (dependencyWait) {
|
||
await resolveDispositionRepairActionAsCovered(
|
||
current,
|
||
"dependency_wait_created",
|
||
);
|
||
return "covered";
|
||
}
|
||
|
||
const state = await collectDispositionRepairSourceState(db, {
|
||
issue: current,
|
||
});
|
||
if (episode) {
|
||
if (!latestRun) return "skipped";
|
||
const decision = await decidePersistedLegacyContinuation(current, latestRun.id, state, episode);
|
||
if (decision.kind === "skip") return "skipped";
|
||
state.fingerprint = legacyDispositionFingerprint(current.companyId, current.id, latestRun.agentId, episode.id);
|
||
}
|
||
if (state.hasActiveExecutionPath) return "skipped";
|
||
if (state.hasDurableWaitingPath) {
|
||
await resolveDispositionRepairActionAsCovered(
|
||
current,
|
||
`durable_path_restored:${state.durablePathReason ?? "unknown"}`,
|
||
);
|
||
return "covered";
|
||
}
|
||
|
||
const ownerAgentId = current.assigneeAgentId;
|
||
const ownerAgent = ownerAgentId ? await getAgent(ownerAgentId) : null;
|
||
const ownerInvokable =
|
||
ownerAgent && ownerAgent.companyId === current.companyId
|
||
? (await isAgentInvokable(ownerAgent)) &&
|
||
isHeartbeatWakeOnDemandEnabled(ownerAgent)
|
||
: false;
|
||
const budgetBlocked = ownerAgentId
|
||
? await isInvocationBudgetBlocked(current, ownerAgentId)
|
||
: true;
|
||
const previousAttempt = readDispositionRepairAttempt(latestRun);
|
||
const activeRepairAction = await recoveryActionsSvc.getActiveForIssue(
|
||
current.companyId,
|
||
current.id,
|
||
);
|
||
// Budget admission may wait behind another recovery worker. Read the
|
||
// execution paths after the persisted counter so a newly reserved
|
||
// successor cannot be mistaken for permission to schedule the next one.
|
||
const currentState = await collectDispositionRepairSourceState(db, { issue: current });
|
||
if (currentState.hasActiveExecutionPath || currentState.hasDurableWaitingPath) return "skipped";
|
||
if (!episode && currentState.fingerprint !== state.fingerprint) return "skipped";
|
||
const runAttempt =
|
||
previousAttempt?.fingerprint === state.fingerprint
|
||
? previousAttempt.attempt
|
||
: 0;
|
||
const persistedAttempt =
|
||
activeRepairAction?.kind === "deliberate_wait_without_target" &&
|
||
activeRepairAction.fingerprint === state.fingerprint
|
||
? activeRepairAction.attemptCount
|
||
: 0;
|
||
// Upgrade compatibility: pre-fingerprint continuation parks already spent
|
||
// attempts against this unchanged source state. Seed the durable counter
|
||
// from that consecutive legacy history instead of granting five fresh
|
||
// attempts merely because the recovery-action row did not exist yet.
|
||
const historicalAttempt = Math.min(
|
||
maxAttempts,
|
||
Math.max(episode?.attempt ?? 0, Math.floor(options.historicalAttemptCount ?? 0)),
|
||
);
|
||
// A source run owns one successor slot. Concurrent checks must not advance
|
||
// the counter again after another checker reserved that same successor.
|
||
if (episode && persistedAttempt > episode.attempt) return "skipped";
|
||
const sameFingerprintAttempt = Math.max(
|
||
runAttempt,
|
||
persistedAttempt,
|
||
historicalAttempt,
|
||
);
|
||
if (episode && (!ownerInvokable || budgetBlocked)) return "skipped";
|
||
if (!ownerInvokable || budgetBlocked) {
|
||
const escalated = await escalateDispositionRepair({
|
||
issue: current,
|
||
latestRun,
|
||
fingerprint: state.fingerprint,
|
||
attemptCount: sameFingerprintAttempt,
|
||
terminalReason: !ownerInvokable
|
||
? "owner_not_invokable"
|
||
: "owner_budget_blocked",
|
||
});
|
||
return escalated ? "escalated" : "skipped";
|
||
}
|
||
|
||
if (sameFingerprintAttempt >= maxAttempts) {
|
||
const escalated = await escalateDispositionRepair({
|
||
issue: current,
|
||
latestRun,
|
||
fingerprint: state.fingerprint,
|
||
attemptCount: sameFingerprintAttempt,
|
||
terminalReason: "unchanged_source_state_exhausted",
|
||
legacyEpisode: episode,
|
||
});
|
||
return escalated ? "escalated" : "skipped";
|
||
}
|
||
|
||
const nextAttempt = sameFingerprintAttempt + 1;
|
||
const action = await ensureDispositionRepairAction({
|
||
issue: current,
|
||
latestRun,
|
||
fingerprint: state.fingerprint,
|
||
attemptCount: sameFingerprintAttempt,
|
||
legacyEpisode: episode,
|
||
});
|
||
const scheduled = await scheduleDispositionRepairAttempt({
|
||
issue: current,
|
||
latestRun,
|
||
action,
|
||
fingerprint: state.fingerprint,
|
||
attempt: nextAttempt,
|
||
legacyEpisode: episode,
|
||
});
|
||
if (!scheduled && episode) {
|
||
// Admission can reject a stale snapshot under the issue lock. An empty
|
||
// repair action must not subsequently cause escalation.
|
||
await recoveryActionsSvc.resolveActiveForIssue({
|
||
companyId: current.companyId, sourceIssueId: current.id, actionId: action.id,
|
||
status: "cancelled", outcome: "cancelled", resolutionNote: "repair_admission_rejected",
|
||
});
|
||
}
|
||
return scheduled ? "queued" : "skipped";
|
||
}
|
||
|
||
async function escalateStrandedAssignedIssue(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
previousStatus: StrandedPreviousStatus;
|
||
latestRun: LatestIssueRun;
|
||
comment?: string;
|
||
notice?: StrandedRecoveryNoticeSeed | null;
|
||
recoveryCause?: StrandedRecoveryCause;
|
||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||
}) {
|
||
if (isStrandedIssueRecoveryIssue(input.issue)) {
|
||
return escalateStrandedRecoveryIssueInPlace({
|
||
issue: input.issue,
|
||
previousStatus: input.previousStatus,
|
||
latestRun: input.latestRun,
|
||
});
|
||
}
|
||
|
||
const recoveryCause = resolveStrandedRecoveryCause(
|
||
input.latestRun,
|
||
input.recoveryCause,
|
||
);
|
||
const recoveryAction = await ensureSourceScopedStrandedRecoveryAction({
|
||
issue: input.issue,
|
||
previousStatus: input.previousStatus,
|
||
latestRun: input.latestRun,
|
||
recoveryCause,
|
||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||
});
|
||
const isProviderQuotaWait =
|
||
recoveryCause === "provider_quota" &&
|
||
!recoveryAction.ownerAgentId &&
|
||
Boolean(recoveryAction.returnOwnerAgentId);
|
||
if (isProviderQuotaWait && recoveryAction.returnOwnerAgentId) {
|
||
await ensureProviderQuotaWaitRecoveryMonitor({
|
||
issue: input.issue,
|
||
latestRun: input.latestRun,
|
||
actionId: recoveryAction.id,
|
||
agentId: recoveryAction.returnOwnerAgentId,
|
||
});
|
||
}
|
||
const blockerIds = await existingUnresolvedBlockerIssueIds(
|
||
input.issue.companyId,
|
||
input.issue.id,
|
||
);
|
||
const updated = await issuesSvc.update(input.issue.id, {
|
||
status: "blocked",
|
||
blockedByIssueIds: blockerIds,
|
||
});
|
||
if (!updated) return null;
|
||
if (isProviderQuotaWait) return updated;
|
||
const sourceAssigneePreserved =
|
||
updated.assigneeAgentId === input.issue.assigneeAgentId &&
|
||
updated.assigneeUserId === input.issue.assigneeUserId;
|
||
|
||
const recoveryOwner = recoveryAction.ownerAgentId
|
||
? await getAgent(recoveryAction.ownerAgentId)
|
||
: null;
|
||
const sourceAssignee = input.issue.assigneeAgentId
|
||
? await getAgent(input.issue.assigneeAgentId)
|
||
: null;
|
||
let notice: SuccessfulRunHandoffNotice | null = null;
|
||
if (
|
||
input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON &&
|
||
input.successfulRunHandoffEvidence
|
||
) {
|
||
const [sourceRun] = input.successfulRunHandoffEvidence.sourceRunId
|
||
? await db
|
||
.select({
|
||
id: heartbeatRuns.id,
|
||
status: heartbeatRuns.status,
|
||
agentId: heartbeatRuns.agentId,
|
||
})
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(
|
||
heartbeatRuns.id,
|
||
input.successfulRunHandoffEvidence.sourceRunId,
|
||
),
|
||
eq(heartbeatRuns.companyId, input.issue.companyId),
|
||
),
|
||
)
|
||
.limit(1)
|
||
: [];
|
||
notice = buildSuccessfulRunHandoffExhaustedNotice({
|
||
issue: input.issue,
|
||
sourceRun: sourceRun ?? null,
|
||
correctiveRun: input.latestRun
|
||
? {
|
||
id: input.latestRun.id,
|
||
status: input.latestRun.status,
|
||
agentId: input.latestRun.agentId,
|
||
}
|
||
: null,
|
||
sourceAssignee,
|
||
recoveryIssue: null,
|
||
recoveryActionId: recoveryAction.id,
|
||
recoveryOwner,
|
||
latestIssueStatus: input.issue.status,
|
||
latestHandoffRunStatus: input.latestRun?.status ?? "unknown",
|
||
missingDisposition:
|
||
input.successfulRunHandoffEvidence.missingDisposition,
|
||
});
|
||
}
|
||
const escalationNotice = buildStrandedRecoveryEscalationNotice({
|
||
seed: input.notice,
|
||
fallbackBody: input.comment,
|
||
recoveryCause,
|
||
recoveryActionId: recoveryAction.id,
|
||
recoveryOwner:
|
||
recoveryAction.ownerAgentId && recoveryOwner
|
||
? { id: recoveryOwner.id, name: recoveryOwner.name }
|
||
: null,
|
||
sourceRun: input.latestRun
|
||
? {
|
||
id: input.latestRun.id,
|
||
agentId: input.latestRun.agentId,
|
||
status: input.latestRun.status,
|
||
errorCode: input.latestRun.errorCode,
|
||
errorSummary: input.latestRun.error
|
||
? redactSensitiveText(input.latestRun.error)
|
||
: null,
|
||
}
|
||
: null,
|
||
});
|
||
|
||
const shouldPostEscalationComment =
|
||
recoveryAction.attemptCount === 1 ||
|
||
input.recoveryCause === "workspace_validation_failed" ||
|
||
input.recoveryCause === "configuration_incomplete";
|
||
if (shouldPostEscalationComment) {
|
||
const escalationCommentMarker = `Recovery action: \`${recoveryAction.id}\``;
|
||
|
||
const hasEscalationComment = await db
|
||
.select({
|
||
id: issueComments.id,
|
||
body: issueComments.body,
|
||
metadata: issueComments.metadata,
|
||
})
|
||
.from(issueComments)
|
||
.where(
|
||
and(
|
||
eq(issueComments.issueId, input.issue.id),
|
||
eq(issueComments.authorType, "system"),
|
||
),
|
||
)
|
||
.orderBy(desc(issueComments.createdAt))
|
||
.limit(50)
|
||
.then((rows) =>
|
||
rows.some(
|
||
(row) =>
|
||
(noticeMetadataReferencesRecoveryAction(
|
||
row.metadata,
|
||
recoveryAction.id,
|
||
) || (row.body ?? "").includes(escalationCommentMarker))
|
||
// Task threads attach notices to runs. A reused incident needs
|
||
// one notice for each failed run so its latest repair stays visible.
|
||
&& (readConfigurationIncompletePayload(input.latestRun)?.reason !== "workspace_base_ref_unresolved"
|
||
|| row.metadata?.sourceRunId === input.latestRun?.id),
|
||
),
|
||
);
|
||
|
||
if (!hasEscalationComment) {
|
||
if (notice) {
|
||
await issuesSvc.addComment(
|
||
input.issue.id,
|
||
notice.body,
|
||
{},
|
||
{
|
||
authorType: "system",
|
||
presentation: notice.presentation,
|
||
metadata: notice.metadata,
|
||
},
|
||
);
|
||
} else {
|
||
await issuesSvc.addComment(
|
||
input.issue.id,
|
||
escalationNotice.body,
|
||
{},
|
||
{
|
||
authorType: "system",
|
||
presentation: escalationNotice.presentation,
|
||
metadata: escalationNotice.metadata,
|
||
},
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
await logActivity(db, {
|
||
companyId: input.issue.companyId,
|
||
actorType: "system",
|
||
actorId: "system",
|
||
agentId: null,
|
||
runId: null,
|
||
action:
|
||
input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
|
||
? "issue.successful_run_handoff_escalated"
|
||
: "issue.updated",
|
||
entityType: "issue",
|
||
entityId: input.issue.id,
|
||
details: {
|
||
identifier: input.issue.identifier,
|
||
status: "blocked",
|
||
previousStatus: input.previousStatus,
|
||
source:
|
||
input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
|
||
? "recovery.reconcile_successful_run_handoff_missing_state"
|
||
: input.recoveryCause === "workspace_validation_failed"
|
||
? "recovery.reconcile_workspace_validation_failed"
|
||
: input.recoveryCause === "configuration_incomplete"
|
||
? "recovery.reconcile_configuration_incomplete"
|
||
: input.recoveryCause ===
|
||
"execution_review_participant_recovery"
|
||
? "recovery.reconcile_execution_review_participant"
|
||
: "recovery.reconcile_stranded_assigned_issue",
|
||
recoveryCause: input.recoveryCause ?? "stranded_assigned_issue",
|
||
latestRunId: input.latestRun?.id ?? null,
|
||
latestRunStatus: input.latestRun?.status ?? null,
|
||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||
recoveryActionId: recoveryAction.id,
|
||
recoveryOwnerType: recoveryAction.ownerType,
|
||
recoveryOwnerAgentId: recoveryAction.ownerAgentId,
|
||
previousOwnerAgentId: recoveryAction.previousOwnerAgentId,
|
||
returnOwnerAgentId: recoveryAction.returnOwnerAgentId,
|
||
routingPolicy:
|
||
parseObject(recoveryAction.evidence).routingPolicy ?? null,
|
||
sourceAssigneeBefore: {
|
||
agentId: input.issue.assigneeAgentId,
|
||
userId: input.issue.assigneeUserId,
|
||
},
|
||
sourceAssigneeAfter: {
|
||
agentId: updated.assigneeAgentId,
|
||
userId: updated.assigneeUserId,
|
||
},
|
||
sourceAssigneePreserved,
|
||
blockerIssueIds: blockerIds,
|
||
},
|
||
});
|
||
|
||
if (!sourceAssigneePreserved) {
|
||
logger.error(
|
||
{
|
||
issueId: input.issue.id,
|
||
beforeAssigneeAgentId: input.issue.assigneeAgentId,
|
||
afterAssigneeAgentId: updated.assigneeAgentId,
|
||
beforeAssigneeUserId: input.issue.assigneeUserId,
|
||
afterAssigneeUserId: updated.assigneeUserId,
|
||
},
|
||
"automatic stranded recovery observed a concurrent source-owner change",
|
||
);
|
||
}
|
||
|
||
return updated;
|
||
}
|
||
|
||
async function persistAdapterFailureRecoveryClassification(
|
||
latestRun: NonNullable<LatestIssueRun>,
|
||
classification: NonNullable<AdapterFailureRecoveryClassification>,
|
||
): Promise<NonNullable<LatestIssueRun>> {
|
||
const classifiedRun = withAdapterFailureRecoveryClassification(
|
||
latestRun,
|
||
classification,
|
||
);
|
||
|
||
const classificationMetadata = withAdapterFailureRecoveryClassification(
|
||
{ ...latestRun, resultJson: {} }, classification,
|
||
).resultJson;
|
||
const [updated] = await db
|
||
.update(heartbeatRuns)
|
||
.set({
|
||
errorCode: classifiedRun.errorCode,
|
||
// Classification owns retry metadata, not workspace repair receipts
|
||
// that may have committed after this run was selected.
|
||
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify(classificationMetadata)}::jsonb`,
|
||
updatedAt: new Date(),
|
||
})
|
||
.where(eq(heartbeatRuns.id, latestRun.id))
|
||
.returning({ resultJson: heartbeatRuns.resultJson, errorCode: heartbeatRuns.errorCode });
|
||
|
||
return updated ? { ...classifiedRun, ...updated } : classifiedRun;
|
||
}
|
||
|
||
function withAdapterFailureRecoveryClassification(
|
||
latestRun: NonNullable<LatestIssueRun>,
|
||
classification: NonNullable<AdapterFailureRecoveryClassification>,
|
||
): NonNullable<LatestIssueRun> {
|
||
const resultJson = parseObject(latestRun.resultJson);
|
||
const providerQuotaMetadata =
|
||
classification.kind === "provider_quota"
|
||
? {
|
||
errorFamily: "provider_quota",
|
||
retryNotBefore: classification.retryAt.toISOString(),
|
||
transientRetryNotBefore: classification.retryAt.toISOString(),
|
||
providerQuotaRetryNotBefore: classification.retryAt.toISOString(),
|
||
}
|
||
: { errorFamily: "configuration_incomplete" };
|
||
const errorCode = classification.kind;
|
||
|
||
return {
|
||
...latestRun,
|
||
errorCode,
|
||
resultJson: {
|
||
...resultJson,
|
||
...providerQuotaMetadata,
|
||
recoveryClassification: errorCode,
|
||
},
|
||
};
|
||
}
|
||
|
||
async function scheduleProviderQuotaRecoveryMonitor(input: {
|
||
issue: typeof issues.$inferSelect;
|
||
latestRun: NonNullable<LatestIssueRun>;
|
||
classification: Extract<
|
||
NonNullable<AdapterFailureRecoveryClassification>,
|
||
{ kind: "provider_quota" }
|
||
>;
|
||
}) {
|
||
if (
|
||
input.issue.status !== "in_progress" &&
|
||
input.issue.status !== "in_review"
|
||
)
|
||
return null;
|
||
|
||
const targetAgentId = getAdapterFailureRecoveryTargetAgentId(input.issue);
|
||
if (!targetAgentId || input.latestRun.agentId !== targetAgentId)
|
||
return null;
|
||
|
||
const previousPolicy = normalizeIssueExecutionPolicy(
|
||
input.issue.executionPolicy ?? null,
|
||
);
|
||
const retryTargetDescription =
|
||
input.issue.status === "in_review"
|
||
? "the active review participant"
|
||
: "the original assignee";
|
||
const policy = {
|
||
...(previousPolicy ?? {
|
||
mode: "normal" as const,
|
||
commentRequired: true,
|
||
stages: [],
|
||
}),
|
||
monitor: {
|
||
nextCheckAt: input.classification.retryAt.toISOString(),
|
||
notes: input.classification.parsedResetTime
|
||
? `Provider usage quota reached; retry ${retryTargetDescription} at the provider reset time.`
|
||
: `Provider usage quota reached; retry ${retryTargetDescription} after the default recovery backoff.`,
|
||
scheduledBy: "assignee" as const,
|
||
kind: "external_service" as const,
|
||
serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME,
|
||
externalRef: input.latestRun.id,
|
||
timeoutAt: null,
|
||
maxAttempts: null,
|
||
recoveryPolicy: "wake_owner" as const,
|
||
},
|
||
};
|
||
const transition = applyIssueMonitorPolicyTransition({
|
||
issue: input.issue,
|
||
policy,
|
||
previousPolicy,
|
||
requestedStatus: input.issue.status,
|
||
requestedAssigneePatch: {},
|
||
actor: { agentId: null, userId: null },
|
||
monitorExplicitlyUpdated: true,
|
||
});
|
||
const updated = await issuesSvc.update(input.issue.id, {
|
||
...transition.patch,
|
||
executionPolicy: policy,
|
||
});
|
||
if (!updated) return null;
|
||
|
||
await logActivity(db, {
|
||
companyId: input.issue.companyId,
|
||
actorType: "system",
|
||
actorId: "recovery",
|
||
agentId: null,
|
||
runId: input.latestRun.id,
|
||
action: "issue.monitor_scheduled",
|
||
entityType: "issue",
|
||
entityId: input.issue.id,
|
||
details: {
|
||
identifier: input.issue.identifier,
|
||
source: "recovery.provider_quota",
|
||
latestRunId: input.latestRun.id,
|
||
errorCode: "provider_quota",
|
||
nextCheckAt: input.classification.retryAt.toISOString(),
|
||
parsedResetTime: input.classification.parsedResetTime,
|
||
targetAgentId,
|
||
},
|
||
});
|
||
|
||
return updated;
|
||
}
|
||
|
||
function getAdapterFailureRecoveryTargetAgentId(
|
||
issue: typeof issues.$inferSelect,
|
||
) {
|
||
if (issue.status !== "in_review") return issue.assigneeAgentId;
|
||
|
||
const pendingExecutionState = parseIssueExecutionState(
|
||
issue.executionState,
|
||
);
|
||
const participant =
|
||
pendingExecutionState?.status === "pending"
|
||
? pendingExecutionState.currentParticipant
|
||
: null;
|
||
return participant?.type === "agent" ? participant.agentId : null;
|
||
}
|
||
|
||
function hasPendingProviderQuotaRecoveryMonitor(
|
||
issue: typeof issues.$inferSelect,
|
||
latestRun: LatestIssueRun,
|
||
now: Date,
|
||
) {
|
||
if (
|
||
!latestRun ||
|
||
!issue.monitorNextCheckAt ||
|
||
issue.monitorNextCheckAt.getTime() <= now.getTime()
|
||
)
|
||
return false;
|
||
const monitor = parseObject(parseObject(issue.executionPolicy).monitor);
|
||
return (
|
||
readNonEmptyString(monitor.serviceName) ===
|
||
PROVIDER_QUOTA_MONITOR_SERVICE_NAME &&
|
||
readNonEmptyString(monitor.externalRef) === latestRun.id
|
||
);
|
||
}
|
||
|
||
async function reconcileStrandedAssignedIssues(opts?: {
|
||
issueCreatedAtGte?: Date | null;
|
||
}) {
|
||
const candidates = await db
|
||
.select()
|
||
.from(issues)
|
||
.where(
|
||
and(
|
||
isNull(issues.assigneeUserId),
|
||
inArray(issues.status, ["todo", "in_progress", "in_review"]),
|
||
or(
|
||
sql`${issues.assigneeAgentId} is not null`,
|
||
eq(issues.status, "in_review"),
|
||
),
|
||
opts?.issueCreatedAtGte
|
||
? gte(issues.createdAt, opts.issueCreatedAtGte)
|
||
: undefined,
|
||
isNull(issues.hiddenAt),
|
||
not(unadmittedChatWakeupCondition(issues.id, issues.companyId)),
|
||
),
|
||
);
|
||
|
||
const result = {
|
||
assignmentDispatched: 0,
|
||
dispatchRequeued: 0,
|
||
continuationRequeued: 0,
|
||
dispositionRepairRequeued: 0,
|
||
productiveContinuationObserved: 0,
|
||
successfulContinuationObserved: 0,
|
||
orphanBlockersAssigned: 0,
|
||
successfulRunHandoffEscalated: 0,
|
||
successfulRunHandoffRetried: 0,
|
||
reviewParticipantRequeued: 0,
|
||
escalated: 0,
|
||
waitingOnReviewResolved: 0,
|
||
providerQuotaMonitored: 0,
|
||
recentProgressExempted: 0,
|
||
operatorCancelExempted: 0,
|
||
onboardingFirstTaskExempted: 0,
|
||
skipped: 0,
|
||
issueIds: [] as string[],
|
||
};
|
||
|
||
const candidateIssueIds = candidates.map((issue) => issue.id);
|
||
const unfinishedGoalBindings = new Set<string>();
|
||
if (candidateIssueIds.length > 0) {
|
||
const pausedGoals = await db
|
||
.select({
|
||
companyId: agentTaskSessions.companyId,
|
||
agentId: agentTaskSessions.agentId,
|
||
taskKey: agentTaskSessions.taskKey,
|
||
})
|
||
.from(agentTaskSessions)
|
||
.where(
|
||
and(
|
||
inArray(agentTaskSessions.taskKey, candidateIssueIds),
|
||
sql`${agentTaskSessions.goalStatus} is not null`,
|
||
sql`${agentTaskSessions.goalStatus} <> 'complete'`,
|
||
),
|
||
);
|
||
for (const goal of pausedGoals) {
|
||
unfinishedGoalBindings.add(
|
||
`${goal.companyId}:${goal.taskKey}:${goal.agentId}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
for (const issue of candidates) {
|
||
if (issue.originKind === "chat_channel") {
|
||
await settleSlackConversation(db, issue.companyId, issue.id);
|
||
const [current] = await db.select({ externalConversationState: externalConversationStateSql() })
|
||
.from(issues).where(and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id)));
|
||
if (current?.externalConversationState === "waiting") { result.skipped += 1; continue; }
|
||
}
|
||
if (issue.conversationAgentId) {
|
||
const lastRun = await getLatestIssueRun(issue.companyId, issue.id);
|
||
if (lastRun?.status === "succeeded") {
|
||
if (await settleConversationTurn(db, (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, lastRun.id)))[0]!)) {
|
||
const [current] = await db.select().from(issues).where(eq(issues.id, issue.id));
|
||
if (current) Object.assign(issue, current);
|
||
}
|
||
}
|
||
if (!(await instanceSettingsService(db).getExperimental()).enableAgentChat) { result.skipped += 1; continue; }
|
||
{
|
||
await deliverConversationComments(db, issue, deps.enqueueWakeup);
|
||
}
|
||
}
|
||
if (isWaitingConversation(issue)) { result.skipped += 1; continue; }
|
||
const executionState = issue.status === "in_review"
|
||
? parseIssueExecutionState(issue.executionState)
|
||
: null;
|
||
const pendingExecutionState = executionState?.status === "pending" ? executionState : null;
|
||
const currentParticipant = pendingExecutionState
|
||
? pendingExecutionState.currentParticipant
|
||
: null;
|
||
const participantAgentId =
|
||
currentParticipant?.type === "agent"
|
||
? currentParticipant.agentId
|
||
: null;
|
||
const agentId =
|
||
issue.status === "in_review" && participantAgentId
|
||
? participantAgentId
|
||
: issue.assigneeAgentId;
|
||
if (!agentId) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
// An unfinished durable session goal owns its continuation lifecycle.
|
||
// A paused goal waits for explicit resume; an active goal is handled by
|
||
// dedicated goal recovery. Generic stranded-work recovery would race
|
||
// either authority and can replace the provider session owning the goal.
|
||
if (
|
||
unfinishedGoalBindings.has(`${issue.companyId}:${issue.id}:${agentId}`)
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
let latestRun = await getLatestIssueRun(issue.companyId, issue.id);
|
||
// Terminal run/wake rows survive a restart even if finalization never
|
||
// reached the escalation transaction. Retry that idempotent disposition
|
||
// before the participant-only review branch can skip pathless reviews.
|
||
if (issue.status === "in_review" && latestRun
|
||
&& isTerminalIssueRun(latestRun)
|
||
&& isIssueReviewPathRecoveryRun(latestRun.contextSnapshot)) {
|
||
const [repairRun] = await db.select().from(heartbeatRuns).where(and(
|
||
eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.id, latestRun.id),
|
||
));
|
||
if (repairRun && await escalateExhaustedIssueReviewPathRecovery(db, { run: repairRun, issueId: issue.id })) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
continue;
|
||
}
|
||
}
|
||
// A native chat can finish between the earlier settlement read and this
|
||
// fresh run read, before its response is materialized. Its trusted
|
||
// finalizer owns that settlement; generic productive-work recovery must
|
||
// not invent another conversation turn during the publication window.
|
||
if (
|
||
issue.conversationAgentId &&
|
||
latestRun?.status === "succeeded" &&
|
||
parseObject(latestRun.resultJson).finalizationReasonCode === "conversation_turn_finished"
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (latestRun?.status === "succeeded" && issue.status !== "in_review") {
|
||
const [source] = await db.select({ runtimeMode: heartbeatRuns.runtimeMode }).from(heartbeatRuns).where(eq(heartbeatRuns.id, latestRun.id)).limit(1);
|
||
if (source?.runtimeMode !== "native") {
|
||
const outcome = await reconcileLegacyContinuation(latestRun.id);
|
||
if (outcome === "queued") {
|
||
result.continuationRequeued += 1;
|
||
result.dispositionRepairRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else if (outcome === "escalated") {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else result.skipped += 1;
|
||
continue;
|
||
}
|
||
}
|
||
|
||
const agent = await getAgent(agentId);
|
||
if (agent?.companyId === issue.companyId && isAgentAwaitingSetup(agent)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
const agentInvokable =
|
||
agent && agent.companyId === issue.companyId
|
||
? await isAgentInvokable(agent)
|
||
: false;
|
||
if (
|
||
agent?.status === "paused" &&
|
||
agent.companyId === issue.companyId &&
|
||
(await hasCurrentNativePassiveWait(issue, latestRun))
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
// A person paused this agent (or stopped its run) on purpose. Escalating its
|
||
// assigned work to `blocked` while the pause lasts fights that decision and
|
||
// strands the issue after resume; leave it for the resumed agent.
|
||
if (
|
||
agent?.status === "paused" &&
|
||
agent.companyId === issue.companyId &&
|
||
latestRun?.agentId === agentId &&
|
||
latestRun.status === "cancelled" &&
|
||
(latestRun.errorCode === "agent_paused" || isOperatorCancelledRun(latestRun, agentId))
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
if (issue.status !== "in_review" && !agentInvokable) {
|
||
const classification = classifyContinuationFailure(latestRun);
|
||
if (
|
||
classification.kind === "deliberate_wait_without_target" ||
|
||
readDispositionRepairAttempt(latestRun)
|
||
) {
|
||
const outcome = await reconcileDispositionRepair(issue, latestRun);
|
||
if (outcome === "escalated") {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
} else {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: issue.status as StrandedPreviousStatus,
|
||
latestRun,
|
||
comment:
|
||
"Paperclip cannot safely continue automatic recovery because the original assignee is not invokable. " +
|
||
"The source assignment is unchanged and the board must choose the next action.",
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (
|
||
await hasActiveExecutionPath(
|
||
issue.companyId,
|
||
issue.id,
|
||
issue.status === "in_review" ? agentId : null,
|
||
)
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (await hasPendingWakeInteraction(issue.companyId, issue.id)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
// A board-owned recovery action is already the durable, human-owned
|
||
// continuation path. Generic stranded-work recovery must not race that
|
||
// authority by launching another provider turn (most importantly after
|
||
// bounded native-session recovery has reached terminal exhaustion).
|
||
const activeRecoveryAction = await recoveryActionsSvc.getActiveForIssue(
|
||
issue.companyId,
|
||
issue.id,
|
||
);
|
||
if (activeRecoveryAction?.ownerType === "board") {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (
|
||
await isAutomaticRecoverySuppressedByPauseHold(
|
||
db,
|
||
issue.companyId,
|
||
issue.id,
|
||
treeControlSvc,
|
||
)
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const participantLatestRunForRecovery =
|
||
issue.status === "in_review" && participantAgentId
|
||
? await getLatestIssueRunForAgent(
|
||
issue.companyId,
|
||
issue.id,
|
||
participantAgentId,
|
||
)
|
||
: null;
|
||
const executionRecoverySource =
|
||
issue.status === "in_review"
|
||
? participantLatestRunForRecovery
|
||
: latestRun;
|
||
if (
|
||
executionRecoverySource?.agentId === agentId &&
|
||
(
|
||
await readChatControlRecoveryStop(db, {
|
||
companyId: issue.companyId,
|
||
issueId: issue.id,
|
||
agentId,
|
||
sourceRunId: executionRecoverySource.id,
|
||
})
|
||
).kind !== "clear"
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
if (isOperatorCancelledRun(executionRecoverySource, agentId)) {
|
||
result.operatorCancelExempted += 1;
|
||
continue;
|
||
}
|
||
if (
|
||
executionRecoverySource &&
|
||
executionRecoverySource.agentId === agentId &&
|
||
["failed", "timed_out", "interrupted", "cancelled"].includes(
|
||
executionRecoverySource.status,
|
||
)
|
||
) {
|
||
const [source] = await db
|
||
.select()
|
||
.from(heartbeatRuns)
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.companyId, issue.companyId),
|
||
eq(heartbeatRuns.id, executionRecoverySource.id),
|
||
),
|
||
);
|
||
if (source && await legacyExecutionNeedsReconciliationWithEvidence(db, source)) {
|
||
await terminalizeLegacyExecution({
|
||
db,
|
||
run: source,
|
||
status: source.status,
|
||
fromStatuses: [source.status],
|
||
});
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
continue;
|
||
}
|
||
}
|
||
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
||
const classification = classifyContinuationFailure(latestRun);
|
||
if (
|
||
classification.kind === "deliberate_wait_without_target" ||
|
||
readDispositionRepairAttempt(latestRun)
|
||
) {
|
||
const outcome = await reconcileDispositionRepair(issue, latestRun);
|
||
if (outcome === "escalated") {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
} else {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: issue.status as StrandedPreviousStatus,
|
||
latestRun,
|
||
recoveryCause:
|
||
issue.status === "in_review"
|
||
? EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON
|
||
: undefined,
|
||
comment:
|
||
"Paperclip cannot safely continue automatic recovery because the original recovery target is over budget. " +
|
||
"The source assignment is unchanged and the board must choose the next action.",
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
const nativeUnblockAction = await nativeBlockedUnblockAction(
|
||
issue,
|
||
latestRun,
|
||
);
|
||
if (nativeUnblockAction) {
|
||
// Older native finalizers left current-track blockers in_progress.
|
||
// A successful provider turn is not permission to discard its pending
|
||
// Board request and recover the original task title as a new objective.
|
||
const updated =
|
||
latestRun &&
|
||
(await repairNativeBlockedWait(
|
||
issue,
|
||
latestRun,
|
||
nativeUnblockAction,
|
||
));
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
if (
|
||
latestRun?.status === "succeeded" &&
|
||
(await hasPersistedDurableWaitPath(issue, latestRun))
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
const recoveryNow = new Date();
|
||
const providerQuotaMonitorRun =
|
||
issue.status === "in_review"
|
||
? participantLatestRunForRecovery
|
||
: latestRun;
|
||
if (
|
||
hasPendingProviderQuotaRecoveryMonitor(
|
||
issue,
|
||
providerQuotaMonitorRun,
|
||
recoveryNow,
|
||
)
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
if (
|
||
isStrandedIssueRecoveryIssue(issue) &&
|
||
isUnsuccessfulTerminalIssueRun(latestRun)
|
||
) {
|
||
const updated = await escalateStrandedRecoveryIssueInPlace({
|
||
issue,
|
||
previousStatus: issue.status as StrandedPreviousStatus,
|
||
latestRun,
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
const adapterFailureClassification =
|
||
issue.status !== "in_review" &&
|
||
latestRun &&
|
||
isUnsuccessfulTerminalIssueRun(latestRun)
|
||
? classifyAdapterFailureForRecovery(latestRun, recoveryNow)
|
||
: null;
|
||
if (latestRun && adapterFailureClassification) {
|
||
const targetAgentId = getAdapterFailureRecoveryTargetAgentId(issue);
|
||
if (!targetAgentId || latestRun.agentId !== targetAgentId) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (adapterFailureClassification.kind === "provider_quota") {
|
||
const monitored = await scheduleProviderQuotaRecoveryMonitor({
|
||
issue,
|
||
latestRun,
|
||
classification: adapterFailureClassification,
|
||
});
|
||
if (monitored) {
|
||
latestRun = await persistAdapterFailureRecoveryClassification(
|
||
latestRun,
|
||
adapterFailureClassification,
|
||
);
|
||
result.providerQuotaMonitored += 1;
|
||
result.issueIds.push(issue.id);
|
||
continue;
|
||
}
|
||
result.skipped += 1;
|
||
continue;
|
||
} else {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: issue.status as StrandedPreviousStatus,
|
||
latestRun,
|
||
recoveryCause: "configuration_incomplete",
|
||
comment:
|
||
"Paperclip classified the latest adapter failure as `configuration_incomplete`. " +
|
||
"Moving the issue to `blocked` with the configuration fix recorded instead of creating a recovery takeover.",
|
||
});
|
||
if (updated) {
|
||
latestRun = await persistAdapterFailureRecoveryClassification(
|
||
latestRun,
|
||
adapterFailureClassification,
|
||
);
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
}
|
||
|
||
const acceptedContinuationInteraction =
|
||
await getLatestAcceptedContinuationInteraction(
|
||
issue.companyId,
|
||
issue.id,
|
||
);
|
||
const acceptedInteractionResolvedAt = acceptedContinuationInteraction
|
||
? (acceptedContinuationInteraction.resolvedAt ??
|
||
acceptedContinuationInteraction.updatedAt)
|
||
: null;
|
||
if (
|
||
acceptedContinuationInteraction &&
|
||
acceptedInteractionResolvedAt &&
|
||
!pendingExecutionState
|
||
) {
|
||
const legacyReviewParkAttempts =
|
||
await summarizeRecentContinuationRetries(
|
||
issue.companyId,
|
||
issue.id,
|
||
agentId,
|
||
CONTINUATION_WAITING_ON_REVIEW_ERROR_CODE,
|
||
acceptedInteractionResolvedAt,
|
||
);
|
||
const successfulRunSinceResolution = await hasSuccessfulIssueRunSince(
|
||
issue.companyId,
|
||
issue.id,
|
||
agentId,
|
||
acceptedInteractionResolvedAt,
|
||
acceptedContinuationInteraction.id,
|
||
);
|
||
|
||
if (!successfulRunSinceResolution) {
|
||
if (!agentInvokable) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (await hasQueuedIssueWake(issue.companyId, issue.id, agentId)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const latestPostResolutionRun = await getLatestIssueRunSince(
|
||
issue.companyId,
|
||
issue.id,
|
||
agentId,
|
||
acceptedInteractionResolvedAt,
|
||
);
|
||
if (
|
||
classifyContinuationFailure(latestPostResolutionRun).kind ===
|
||
"deliberate_wait_without_target"
|
||
) {
|
||
const resolved = await resolveContinuationWaitingOnReview(issue);
|
||
if (resolved) {
|
||
result.waitingOnReviewResolved += 1;
|
||
result.issueIds.push(issue.id);
|
||
continue;
|
||
}
|
||
const outcome = await reconcileDispositionRepair(
|
||
issue,
|
||
latestPostResolutionRun,
|
||
{
|
||
historicalAttemptCount: legacyReviewParkAttempts.consecutive,
|
||
},
|
||
);
|
||
if (outcome === "queued") {
|
||
result.continuationRequeued += 1;
|
||
result.dispositionRepairRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else if (outcome === "escalated") {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
const { consecutive } = legacyReviewParkAttempts;
|
||
if (
|
||
consecutive >= INTERACTION_CONTINUATION_REQUEUE_MAX_ATTEMPTS &&
|
||
latestPostResolutionRun
|
||
) {
|
||
const resolved = await resolveContinuationWaitingOnReview(issue);
|
||
if (resolved) {
|
||
result.waitingOnReviewResolved += 1;
|
||
result.issueIds.push(issue.id);
|
||
continue;
|
||
}
|
||
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: issue.status as StrandedPreviousStatus,
|
||
latestRun: latestPostResolutionRun,
|
||
comment:
|
||
`Paperclip stopped requeueing accepted interaction \`${acceptedContinuationInteraction.id}\` after ` +
|
||
`${consecutive} consecutive continuation wakes were cancelled while waiting on review. ` +
|
||
"Moving the issue to `blocked` so the missing execution path is visible for intervention.",
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
const queued = await enqueueStrandedIssueRecovery({
|
||
issueId: issue.id,
|
||
agentId,
|
||
reason: "issue_continuation_needed",
|
||
retryReason: "issue_continuation_needed",
|
||
source: "issue.interaction_continuation_recovery",
|
||
retryOfRunId:
|
||
latestPostResolutionRun?.id ??
|
||
acceptedContinuationInteraction.sourceRunId ??
|
||
latestRun?.id ??
|
||
null,
|
||
extraContext: {
|
||
mutation: "interaction",
|
||
interactionId: acceptedContinuationInteraction.id,
|
||
interactionKind: acceptedContinuationInteraction.kind,
|
||
interactionStatus: acceptedContinuationInteraction.status,
|
||
interactionContinuationPolicy:
|
||
acceptedContinuationInteraction.continuationPolicy,
|
||
interactionResolvedAt:
|
||
acceptedInteractionResolvedAt.toISOString(),
|
||
},
|
||
});
|
||
if (queued) {
|
||
result.continuationRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
}
|
||
|
||
if (issue.status === "in_review") {
|
||
if (!participantAgentId || !pendingExecutionState) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
const participantLatestRun = participantLatestRunForRecovery;
|
||
|
||
if (
|
||
!participantLatestRun ||
|
||
!isTerminalIssueRun(participantLatestRun)
|
||
) {
|
||
if (!agentInvokable) {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_review",
|
||
latestRun: participantLatestRun,
|
||
notice: buildExecutionReviewParticipantUnavailableNoticeSeed(),
|
||
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
const participantAdapterFailureClassification =
|
||
isUnsuccessfulTerminalIssueRun(participantLatestRun)
|
||
? classifyAdapterFailureForRecovery(
|
||
participantLatestRun,
|
||
recoveryNow,
|
||
)
|
||
: null;
|
||
if (
|
||
participantAdapterFailureClassification?.kind === "provider_quota"
|
||
) {
|
||
const monitored = await scheduleProviderQuotaRecoveryMonitor({
|
||
issue,
|
||
latestRun: participantLatestRun,
|
||
classification: participantAdapterFailureClassification,
|
||
});
|
||
if (monitored) {
|
||
latestRun = await persistAdapterFailureRecoveryClassification(
|
||
participantLatestRun,
|
||
participantAdapterFailureClassification,
|
||
);
|
||
result.providerQuotaMonitored += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
if (
|
||
participantAdapterFailureClassification?.kind ===
|
||
"configuration_incomplete"
|
||
) {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_review",
|
||
latestRun: participantLatestRun,
|
||
recoveryCause: "configuration_incomplete",
|
||
comment:
|
||
"Paperclip classified the active review participant's latest adapter failure as " +
|
||
"`configuration_incomplete`. Moving the issue to `blocked` with the configuration fix " +
|
||
"recorded instead of repeatedly requeueing the reviewer.",
|
||
});
|
||
if (updated) {
|
||
latestRun = await persistAdapterFailureRecoveryClassification(
|
||
participantLatestRun,
|
||
participantAdapterFailureClassification,
|
||
);
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (!agentInvokable) {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_review",
|
||
latestRun: participantLatestRun,
|
||
notice: buildExecutionReviewParticipantUnavailableNoticeSeed(),
|
||
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (
|
||
didAutomaticRecoveryFail(
|
||
participantLatestRun,
|
||
EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
|
||
)
|
||
) {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_review",
|
||
latestRun: participantLatestRun,
|
||
notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
|
||
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (
|
||
await hasQueuedIssueWake(
|
||
issue.companyId,
|
||
issue.id,
|
||
participantAgentId,
|
||
)
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (await isInvocationBudgetBlocked(issue, participantAgentId)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const reviewOutcome: { retryExhausted?: boolean } = {};
|
||
const queued = await enqueueStrandedIssueRecovery({
|
||
issueId: issue.id,
|
||
agentId: participantAgentId,
|
||
reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
|
||
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
|
||
source: "issue.execution_review_recovery",
|
||
retryOfRunId: participantLatestRun.id,
|
||
extraContext: {
|
||
currentStageId: pendingExecutionState.currentStageId ?? null,
|
||
currentStageType: pendingExecutionState.currentStageType ?? null,
|
||
reviewRecoveryInstruction:
|
||
"The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.",
|
||
},
|
||
outcome: reviewOutcome,
|
||
});
|
||
if (queued) {
|
||
result.reviewParticipantRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else if (
|
||
reviewOutcome.retryExhausted &&
|
||
!(await hasActiveExecutionPath(issue.companyId, issue.id, null))
|
||
) {
|
||
// Same exhaustion as the other lanes: the reviewer run's bounded
|
||
// retries are spent, so escalate as the review-recovery failure it
|
||
// is instead of skipping on every sweep with no live path. The
|
||
// live-path re-read guards the write against a run or wake that
|
||
// started after the loop's check — for ANY agent, not only the
|
||
// participant: `blocked` is issue-wide, so another agent still
|
||
// working the issue must never be blocked under.
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_review",
|
||
latestRun: participantLatestRun,
|
||
notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
|
||
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (issue.status === "todo") {
|
||
if (!latestRun) {
|
||
// The onboarding first task is deliberately created without a wake:
|
||
// nothing runs and no token is spent until the user types. It is not
|
||
// stranded work, so liveness dispatch must leave it alone until a
|
||
// user comment exists (that comment wakes the assignee through the
|
||
// normal comment path, and only then may recovery treat a lost wake
|
||
// as stranded).
|
||
if (await isOnboardingFirstTaskAwaitingUser(issue)) {
|
||
result.onboardingFirstTaskExempted += 1;
|
||
continue;
|
||
}
|
||
|
||
if (await hasQueuedIssueWake(issue.companyId, issue.id)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const queued = await enqueueInitialAssignedTodoDispatch(
|
||
issue,
|
||
agentId,
|
||
);
|
||
if (queued) {
|
||
result.assignmentDispatched += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (
|
||
latestRun.status === "succeeded" &&
|
||
!(await wasTodoHandedBackDuringOrAfterLatestRun(issue, latestRun))
|
||
) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (didAutomaticRecoveryFail(latestRun, "assignment_recovery")) {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "todo",
|
||
latestRun,
|
||
notice: {
|
||
body:
|
||
"Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " +
|
||
"but it still has no live execution path. " +
|
||
"Moving it to `blocked` so it is visible for intervention.",
|
||
title: "No live execution path",
|
||
tone: "danger",
|
||
},
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const dispatchOutcome: { retryExhausted?: boolean } = {};
|
||
const queued = await enqueueStrandedIssueRecovery({
|
||
issueId: issue.id,
|
||
agentId,
|
||
reason: "issue_assignment_recovery",
|
||
retryReason: "assignment_recovery",
|
||
source: "issue.assignment_recovery",
|
||
retryOfRunId: latestRun.id,
|
||
outcome: dispatchOutcome,
|
||
});
|
||
if (queued) {
|
||
result.dispatchRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else if (
|
||
dispatchOutcome.retryExhausted &&
|
||
!(await hasActiveExecutionPath(issue.companyId, issue.id, null))
|
||
) {
|
||
// Same exhaustion as the in_progress lane: the lost dispatch's
|
||
// bounded retries are spent, so escalate instead of skipping on
|
||
// every sweep with no live path. The live-path re-read guards the
|
||
// `blocked` write against a run or wake that started after the
|
||
// loop's check.
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "todo",
|
||
latestRun,
|
||
notice: {
|
||
body:
|
||
"Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " +
|
||
"but the bounded retry budget is spent and it still has no live execution path. " +
|
||
"Moving it to `blocked` so it is visible for intervention.",
|
||
title: "No live execution path",
|
||
tone: "danger",
|
||
},
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (!latestRun && !issue.checkoutRunId && !issue.executionRunId) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
if (readDispositionRepairAttempt(latestRun)) {
|
||
const outcome = await reconcileDispositionRepair(issue, latestRun);
|
||
if (outcome === "queued") {
|
||
result.continuationRequeued += 1;
|
||
result.dispositionRepairRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else if (outcome === "escalated") {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
const handoffEvidence = isExhaustedSuccessfulRunHandoff(latestRun);
|
||
if (handoffEvidence) {
|
||
if (isPluginManagedIssueLifecycle(issue)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
if (!handoffEvidence.exhausted) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
// An interrupted corrective run is not evidence that the agent could
|
||
// not choose a disposition: a graceful server shutdown (a deploy
|
||
// restart, a lost process) ended the attempt before the agent
|
||
// finished. The attempt cap counts attempts the agent got to finish,
|
||
// so give the interrupted run the same bounded transient retry any
|
||
// interrupted run gets — the retry keeps the handoff context, so it
|
||
// is still the corrective run — and escalate only once that retry
|
||
// budget is spent or a finished attempt still leaves no disposition.
|
||
// Native-runtime corrective runs have no process-loss retry lane
|
||
// (a graceful shutdown suspends their controller for reattach
|
||
// instead of interrupting them; other native interruptions are
|
||
// reconciled by their own finalizer), so the recovery enqueue
|
||
// returns null for them and they escalate exactly as before.
|
||
if (latestRun?.status === "interrupted") {
|
||
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
const retried = await enqueueStrandedIssueRecovery({
|
||
issueId: issue.id,
|
||
agentId,
|
||
reason: "issue_continuation_needed",
|
||
retryReason: "issue_continuation_needed",
|
||
source: "issue.successful_run_handoff_interrupted_retry",
|
||
retryOfRunId: latestRun.id,
|
||
});
|
||
if (retried) {
|
||
result.successfulRunHandoffRetried += 1;
|
||
result.issueIds.push(issue.id);
|
||
continue;
|
||
}
|
||
}
|
||
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_progress",
|
||
latestRun,
|
||
recoveryCause: SUCCESSFUL_RUN_MISSING_STATE_REASON,
|
||
successfulRunHandoffEvidence: handoffEvidence,
|
||
});
|
||
if (updated) {
|
||
result.successfulRunHandoffEscalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
if (isSuccessfulInProgressContinuationRun(latestRun)) {
|
||
const successfulRun = latestRun;
|
||
|
||
// A child with a live or durable waiting path must get a chance to use
|
||
// the shared workspace. Repeated automatic parent continuations can
|
||
// otherwise reacquire it before the child's resource retry is due.
|
||
// This only gates recovery; explicit messages still follow admission.
|
||
const workspace = parseObject(parseObject(successfulRun.contextSnapshot).paperclipWorkspace);
|
||
if (workspace.mode === "shared_workspace" && (await healthyOpenChildIssues(issue, true)).length > 0) {
|
||
result.productiveContinuationObserved += 1;
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (!isProductiveContinuationRun(successfulRun)) {
|
||
result.successfulContinuationObserved += 1;
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (isRepeatedProductiveContinuationRecovery(successfulRun)) {
|
||
// GGU-809: skip escalation if the assignee has shown visible progress
|
||
// (comment or attachment) within the exemption window. Falling
|
||
// through here lets the normal continuation-retry path enqueue the
|
||
// next wake, which is the correct behaviour for batch workflows.
|
||
const exempted = await hasRecentVisibleProgress(
|
||
issue.companyId,
|
||
issue.id,
|
||
agentId,
|
||
STRANDED_RECENT_PROGRESS_EXEMPTION_MS,
|
||
);
|
||
if (!exempted) {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_progress",
|
||
latestRun: successfulRun,
|
||
comment:
|
||
"Paperclip automatically retried continuation for this assigned `in_progress` issue and the retry " +
|
||
"made progress, but it still has no live execution path. Moving it to `blocked` so it is visible for intervention.",
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
result.recentProgressExempted += 1;
|
||
}
|
||
|
||
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const queued = await enqueueStrandedIssueRecovery({
|
||
issueId: issue.id,
|
||
agentId,
|
||
reason: "issue_continuation_needed",
|
||
retryReason: "issue_continuation_needed",
|
||
source: "issue.productive_terminal_continuation_recovery",
|
||
retryOfRunId: successfulRun.id,
|
||
});
|
||
if (queued) {
|
||
result.continuationRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
if (isUnsuccessfulTerminalIssueRun(latestRun)) {
|
||
const classification = classifyContinuationFailure(latestRun);
|
||
|
||
if (
|
||
classification.errorCode === CONTINUATION_WAITING_ON_REVIEW_ERROR_CODE
|
||
) {
|
||
const resolved = await resolveContinuationWaitingOnReview(issue);
|
||
if (resolved) {
|
||
result.waitingOnReviewResolved += 1;
|
||
result.issueIds.push(issue.id);
|
||
continue;
|
||
}
|
||
|
||
const outcome = await reconcileDispositionRepair(issue, latestRun);
|
||
if (outcome === "queued") {
|
||
result.continuationRequeued += 1;
|
||
result.dispositionRepairRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else if (outcome === "escalated") {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (classification.kind === "non_retryable") {
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_progress",
|
||
latestRun,
|
||
notice: {
|
||
body:
|
||
"Paperclip detected a non-retryable failure on this issue's continuation run " +
|
||
`(\`${classification.errorCode}\`). Skipping automatic retries and moving it to \`blocked\` ` +
|
||
"so it is visible for intervention.",
|
||
title: "Continuation failed",
|
||
tone: "danger",
|
||
},
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (didAutomaticRecoveryFail(latestRun, "issue_continuation_needed")) {
|
||
const { consecutive, latestFinishedAt } =
|
||
await summarizeRecentContinuationRetries(
|
||
issue.companyId,
|
||
issue.id,
|
||
agentId,
|
||
classification.errorCode,
|
||
);
|
||
if (consecutive >= classification.maxAttempts) {
|
||
const attemptCopy =
|
||
consecutive <= 1 ? "" : ` (${consecutive}× attempts)`;
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_progress",
|
||
latestRun,
|
||
notice: {
|
||
body:
|
||
"Paperclip automatically retried continuation for this assigned `in_progress` issue after its live " +
|
||
`execution disappeared, but it still has no live execution path${attemptCopy}. ` +
|
||
"Moving it to `blocked` so it is visible for intervention.",
|
||
title: "No live execution path",
|
||
tone: "danger",
|
||
},
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (classification.baseBackoffMs > 0 && latestFinishedAt) {
|
||
const elapsed = Date.now() - latestFinishedAt.getTime();
|
||
const requiredDelay =
|
||
classification.baseBackoffMs *
|
||
Math.pow(2, Math.max(0, consecutive - 1));
|
||
if (elapsed < requiredDelay) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
||
result.skipped += 1;
|
||
continue;
|
||
}
|
||
|
||
const recoveryOutcome: { retryExhausted?: boolean } = {};
|
||
const queued = await enqueueStrandedIssueRecovery({
|
||
issueId: issue.id,
|
||
agentId,
|
||
reason: "issue_continuation_needed",
|
||
retryReason: "issue_continuation_needed",
|
||
source: "issue.continuation_recovery",
|
||
retryOfRunId: latestRun?.id ?? issue.checkoutRunId ?? null,
|
||
outcome: recoveryOutcome,
|
||
});
|
||
if (queued) {
|
||
result.continuationRequeued += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else if (
|
||
recoveryOutcome.retryExhausted &&
|
||
!(await hasActiveExecutionPath(issue.companyId, issue.id, null))
|
||
) {
|
||
// The failed run's bounded transient retries are all spent, so no
|
||
// successor will ever be queued for it. Escalate rather than skip
|
||
// on every sweep forever: the issue would otherwise stay
|
||
// `in_progress` with no run and no path until a person noticed.
|
||
// The live-path re-read guards the `blocked` write against a run or
|
||
// wake that started after the loop's check.
|
||
const updated = await escalateStrandedAssignedIssue({
|
||
issue,
|
||
previousStatus: "in_progress",
|
||
latestRun,
|
||
notice: {
|
||
body:
|
||
"Paperclip retried this issue's run after it ended without finishing, but the bounded retry budget " +
|
||
"is spent and it still has no live execution path. " +
|
||
"Moving it to `blocked` so it is visible for intervention.",
|
||
title: "No live execution path",
|
||
tone: "danger",
|
||
},
|
||
});
|
||
if (updated) {
|
||
result.escalated += 1;
|
||
result.issueIds.push(issue.id);
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
} else {
|
||
result.skipped += 1;
|
||
}
|
||
}
|
||
|
||
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
|
||
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
|
||
result.skipped += orphanBlockerRecovery.skipped;
|
||
result.issueIds.push(...orphanBlockerRecovery.issueIds);
|
||
|
||
const activeRecovery = await reconcileActiveRecoveryActions();
|
||
result.continuationRequeued += activeRecovery.requeued;
|
||
result.escalated += activeRecovery.escalated;
|
||
result.skipped += activeRecovery.skipped;
|
||
result.issueIds.push(...activeRecovery.issueIds);
|
||
result.issueIds = [...new Set(result.issueIds)];
|
||
|
||
return result;
|
||
}
|
||
|
||
async function reconcileResolvedDependencyWakeBackstop(
|
||
opts?: ResolvedDependencyWakeBackstopOptions,
|
||
) {
|
||
const result = {
|
||
checked: 0,
|
||
healed: 0,
|
||
existingWakeSkipped: 0,
|
||
selfBlockedSkipped: 0,
|
||
livePathSkipped: 0,
|
||
interactionSkipped: 0,
|
||
pauseHoldSkipped: 0,
|
||
notReadySkipped: 0,
|
||
candidateLimitSkipped: 0,
|
||
deferredOrFailed: 0,
|
||
enqueueFailed: 0,
|
||
issueIds: [] as string[],
|
||
};
|
||
|
||
const source = opts?.source ?? "issue_graph_liveness.backstop";
|
||
const requestedByActorId =
|
||
source === "workspace.finalize"
|
||
? "heartbeat_finalize"
|
||
: "issue_graph_liveness_backstop";
|
||
const payloadBackstop =
|
||
source === "workspace.finalize"
|
||
? "workspace_finalize_reconciliation"
|
||
: "issue_graph_liveness_reconciliation";
|
||
const useCursor = !opts?.blockerIssueId;
|
||
|
||
const queryCandidates = (afterIssueId: string | null) => {
|
||
const filters = [
|
||
eq(issues.status, "blocked"),
|
||
isNull(issues.conversationAgentId),
|
||
visibleIssueCondition(),
|
||
sql`${issues.assigneeAgentId} is not null`,
|
||
];
|
||
if (opts?.companyId) filters.push(eq(issues.companyId, opts.companyId));
|
||
if (afterIssueId) filters.push(gt(issues.id, afterIssueId));
|
||
|
||
if (opts?.blockerIssueId) {
|
||
filters.push(
|
||
eq(issueRelations.companyId, issues.companyId),
|
||
eq(issueRelations.type, "blocks"),
|
||
eq(issueRelations.issueId, opts.blockerIssueId),
|
||
eq(issueRelations.relatedIssueId, issues.id),
|
||
);
|
||
return db
|
||
.select({
|
||
id: issues.id,
|
||
companyId: issues.companyId,
|
||
identifier: issues.identifier,
|
||
assigneeAgentId: issues.assigneeAgentId,
|
||
blockedTransitionAt: issues.blockedTransitionAt,
|
||
totalCount: sql<number>`count(*) over()::int`,
|
||
})
|
||
.from(issueRelations)
|
||
.innerJoin(issues, eq(issueRelations.relatedIssueId, issues.id))
|
||
.where(and(...filters))
|
||
.orderBy(asc(issues.id))
|
||
.limit(RESOLVED_DEPENDENCY_WAKE_BACKSTOP_CANDIDATE_LIMIT);
|
||
}
|
||
|
||
return db
|
||
.select({
|
||
id: issues.id,
|
||
companyId: issues.companyId,
|
||
identifier: issues.identifier,
|
||
assigneeAgentId: issues.assigneeAgentId,
|
||
blockedTransitionAt: issues.blockedTransitionAt,
|
||
totalCount: sql<number>`count(*) over()::int`,
|
||
})
|
||
.from(issues)
|
||
.where(and(...filters))
|
||
.orderBy(asc(issues.id))
|
||
.limit(RESOLVED_DEPENDENCY_WAKE_BACKSTOP_CANDIDATE_LIMIT);
|
||
};
|
||
|
||
let candidateRows = await queryCandidates(
|
||
useCursor ? resolvedDependencyWakeBackstopCandidateCursor : null,
|
||
);
|
||
if (
|
||
useCursor &&
|
||
candidateRows.length === 0 &&
|
||
resolvedDependencyWakeBackstopCandidateCursor
|
||
) {
|
||
resolvedDependencyWakeBackstopCandidateCursor = null;
|
||
candidateRows = await queryCandidates(null);
|
||
}
|
||
const totalCandidateCount = candidateRows[0]?.totalCount ?? 0;
|
||
const candidates = candidateRows.map(
|
||
({ totalCount: _totalCount, ...candidate }) => candidate,
|
||
);
|
||
result.checked = candidates.length;
|
||
result.candidateLimitSkipped = Math.max(
|
||
0,
|
||
totalCandidateCount - candidates.length,
|
||
);
|
||
const lastCandidate = candidates[candidates.length - 1] ?? null;
|
||
if (useCursor) {
|
||
resolvedDependencyWakeBackstopCandidateCursor =
|
||
result.candidateLimitSkipped > 0 && lastCandidate
|
||
? lastCandidate.id
|
||
: null;
|
||
}
|
||
if (result.candidateLimitSkipped > 0) {
|
||
logger.warn(
|
||
{
|
||
processed: candidates.length,
|
||
skipped: result.candidateLimitSkipped,
|
||
limit: RESOLVED_DEPENDENCY_WAKE_BACKSTOP_CANDIDATE_LIMIT,
|
||
nextCursor: useCursor
|
||
? resolvedDependencyWakeBackstopCandidateCursor
|
||
: null,
|
||
source,
|
||
blockerIssueId: opts?.blockerIssueId ?? null,
|
||
},
|
||
"issue graph liveness backstop deferred resolved dependency wake candidates past page limit",
|
||
);
|
||
}
|
||
|
||
const candidatesByCompany = new Map<string, typeof candidates>();
|
||
for (const candidate of candidates) {
|
||
const companyCandidates =
|
||
candidatesByCompany.get(candidate.companyId) ?? [];
|
||
companyCandidates.push(candidate);
|
||
candidatesByCompany.set(candidate.companyId, companyCandidates);
|
||
}
|
||
|
||
for (const [
|
||
companyId,
|
||
companyCandidates,
|
||
] of candidatesByCompany.entries()) {
|
||
const readinessMap = await issuesSvc.listDependencyReadiness(
|
||
companyId,
|
||
companyCandidates.map((candidate) => candidate.id),
|
||
);
|
||
|
||
for (const candidate of companyCandidates) {
|
||
const agentId = candidate.assigneeAgentId;
|
||
if (!agentId) continue;
|
||
|
||
const readiness = readinessMap.get(candidate.id);
|
||
const resolvedBlockerIssueId = readiness?.blockerIssueIds[0] ?? null;
|
||
if (
|
||
!readiness ||
|
||
!readiness.isDependencyReady ||
|
||
readiness.blockerIssueIds.length === 0 ||
|
||
!resolvedBlockerIssueId
|
||
) {
|
||
result.notReadySkipped += 1;
|
||
continue;
|
||
}
|
||
|
||
// Level-triggered dedup: key on the full blocker set (the current ready
|
||
// state), not on any single resolved edge. An older completed per-edge
|
||
// wake for an earlier partial resolution has a different key, so it does
|
||
// not suppress this wake. The shared helper still suppresses a duplicate
|
||
// wake for the SAME ready state, which bounds reconciliation.
|
||
const idempotencyKey = buildIssueBlockersResolvedWakeStateKey({
|
||
dependentIssueId: candidate.id,
|
||
blockerIssueIds: readiness.blockerIssueIds,
|
||
blockedTransitionAt: candidate.blockedTransitionAt,
|
||
});
|
||
const existingWake =
|
||
await findExistingIssueBlockersResolvedWakeForReadyState(db, {
|
||
companyId,
|
||
dependentIssueId: candidate.id,
|
||
blockerIssueIds: readiness.blockerIssueIds,
|
||
blockedTransitionAt: candidate.blockedTransitionAt,
|
||
});
|
||
if (existingWake) {
|
||
result.existingWakeSkipped += 1;
|
||
continue;
|
||
}
|
||
// The assignee agent opened this blocked cycle itself, knowing its recorded
|
||
// blockers were resolved: it is waiting on something outside the blocker
|
||
// graph. Re-waking it here made it re-block on every sweep (each re-block
|
||
// stamps a new cycle, so the ready-state key never repeats).
|
||
if (await wasBlockedByAssigneeAgent(companyId, candidate.id, agentId, candidate.blockedTransitionAt, readiness.blockerIssueIds)) {
|
||
result.selfBlockedSkipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (
|
||
(await hasActiveExecutionPath(companyId, candidate.id, agentId)) ||
|
||
(await hasQueuedIssueWake(companyId, candidate.id, agentId))
|
||
) {
|
||
result.livePathSkipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (await hasPendingWakeInteraction(companyId, candidate.id)) {
|
||
result.interactionSkipped += 1;
|
||
continue;
|
||
}
|
||
|
||
if (
|
||
await isAutomaticRecoverySuppressedByPauseHold(
|
||
db,
|
||
companyId,
|
||
candidate.id,
|
||
treeControlSvc,
|
||
)
|
||
) {
|
||
result.pauseHoldSkipped += 1;
|
||
continue;
|
||
}
|
||
|
||
try {
|
||
const wake = await deps.enqueueWakeup(agentId, {
|
||
source: "automation",
|
||
triggerDetail: "system",
|
||
reason: ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
|
||
payload: {
|
||
issueId: candidate.id,
|
||
resolvedBlockerIssueId,
|
||
blockerIssueIds: readiness.blockerIssueIds,
|
||
backstop: payloadBackstop,
|
||
},
|
||
idempotencyKey,
|
||
requestedByActorType: "system",
|
||
requestedByActorId,
|
||
contextSnapshot: {
|
||
issueId: candidate.id,
|
||
taskId: candidate.id,
|
||
wakeReason: ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
|
||
source,
|
||
resolvedBlockerIssueId,
|
||
blockerIssueIds: readiness.blockerIssueIds,
|
||
},
|
||
});
|
||
if (!wake) {
|
||
// enqueueWakeup returns null for normal deferred/skipped paths
|
||
// such as disabled wake-on-demand or concurrency gating. That is
|
||
// not an enqueue error, but the backstop still did not heal now.
|
||
result.deferredOrFailed += 1;
|
||
continue;
|
||
}
|
||
|
||
result.healed += 1;
|
||
result.issueIds.push(candidate.id);
|
||
|
||
await logActivity(db, {
|
||
companyId,
|
||
actorType: "system",
|
||
actorId: "issue_graph_liveness_backstop",
|
||
agentId,
|
||
runId: opts?.runId ?? null,
|
||
action: "issue.blockers_resolved_wake_emitted",
|
||
entityType: "issue",
|
||
entityId: candidate.id,
|
||
details: {
|
||
source,
|
||
wakeupRunId: wake.id,
|
||
idempotencyKey,
|
||
resolvedBlockerIssueId,
|
||
blockerIssueIds: readiness.blockerIssueIds,
|
||
},
|
||
});
|
||
} catch (err) {
|
||
result.deferredOrFailed += 1;
|
||
result.enqueueFailed += 1;
|
||
logger.warn(
|
||
{ err, issueId: candidate.id, agentId, idempotencyKey, source },
|
||
"failed to enqueue dependency wake from issue graph liveness backstop",
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (result.healed > 0) {
|
||
logger.warn(
|
||
{
|
||
healed: result.healed,
|
||
issueIds: result.issueIds,
|
||
source,
|
||
blockerIssueId: opts?.blockerIssueId ?? null,
|
||
},
|
||
"issue graph liveness backstop healed resolved blocked dependency wakes",
|
||
);
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
function readRecoveryTimerIntervalMs(raw: unknown, fallback: number) {
|
||
return Math.max(1, Math.floor(asNumber(raw, fallback)));
|
||
}
|
||
|
||
// Backstop reconciler: terminalizes a "running" run that can no longer reach a
|
||
// terminal status on its own. The run finalizer writes the terminal status in
|
||
// a step that is separate from the agent status=done PATCH. When the teardown
|
||
// stops between the two steps, heartbeat_runs.status stays "running" forever.
|
||
// The UI reads liveness from that row, so the task shows "Live" forever. This
|
||
// function forces the run to a terminal status and records a run event, so the
|
||
// state is auditable. It never overwrites a status that another path already
|
||
// made terminal.
|
||
//
|
||
// A live controller lease owns execution and finalization across server
|
||
// processes. Only after that ownership ends can either authority below
|
||
// terminalize the run:
|
||
//
|
||
// - Issue-terminal authority: the run's issue already reached a terminal
|
||
// status (done or cancelled), but the run row is still "running". A healthy
|
||
// run always terminalizes its own row before or just after the issue reaches
|
||
// a terminal status, so a lasting "running" row under a terminal issue is
|
||
// orphaned. This authority does not depend on process death. It is the only
|
||
// authority that catches the reuse-lease path: the release stops the sandbox
|
||
// but keeps the server process alive, so the in-memory handle and the
|
||
// recorded pid can both persist.
|
||
// - Process-death authority: the run has no in-memory handle and its recorded
|
||
// process and process group are both gone. This catches a hard server crash
|
||
// that skipped the graceful teardown, even when the issue is not terminal.
|
||
async function terminalizeOrphanedRunningRun(
|
||
run: typeof heartbeatRuns.$inferSelect,
|
||
options?: {
|
||
// The terminal run status implied by a referencing issue. The caller
|
||
// passes it when it already knows the issue that holds the run in a lock
|
||
// column. It maps issue "done" to "succeeded" and issue "cancelled" to
|
||
// "cancelled". A null value means the referencing issue is not terminal.
|
||
referencingIssueTerminalStatus?: "succeeded" | "cancelled" | null;
|
||
// True when an active (non-terminal) issue still holds this run in a lock
|
||
// column. The run is live for that active issue, so the caller forbids the
|
||
// issue-terminal authority. This flag also suppresses the context-snapshot
|
||
// fallback below. Without it, a terminal issue named in the run context
|
||
// snapshot would still terminalize the shared run and defeat the guard.
|
||
runReferencedByActiveIssue?: boolean;
|
||
},
|
||
): Promise<{ terminalized: boolean; status: string }> {
|
||
// Act only on a run in "running" status. A "queued" run has no process yet,
|
||
// and a "scheduled_retry" run has no process on purpose because it waits to
|
||
// retry. Neither is orphaned, so this function must not terminalize them.
|
||
if (run.status !== "running")
|
||
return { terminalized: false, status: run.status };
|
||
// Authentication failure does not prove the retained provider stopped.
|
||
// PID observations and task status edits cannot resolve its ownership.
|
||
if (isNativeRunnerOwnershipHeld(run))
|
||
return { terminalized: false, status: run.status };
|
||
|
||
// Another controller may own a sandbox run whose PID has no meaning on
|
||
// this host. Its live lease owns both execution and finalization, even if
|
||
// the issue is already terminal or this process has no in-memory handle.
|
||
if (await hasLiveLegacyController(db, run))
|
||
return { terminalized: false, status: run.status };
|
||
|
||
const pid = run.processPid ?? null;
|
||
const processGroupId = run.processGroupId ?? null;
|
||
|
||
// Issue-terminal authority. When the run's issue is terminal, the run row is
|
||
// orphaned regardless of process or handle state. Prefer the referencing
|
||
// issue status that the caller passed, because a lock column is the direct
|
||
// link from the stuck "Live" issue to this run. Fall back to the issue id in
|
||
// the run context snapshot when the caller passed nothing. Skip the fallback
|
||
// when an active issue still references the run. The run is live for that
|
||
// active issue, so a terminal issue named in the context snapshot must not
|
||
// terminalize it.
|
||
const runContext = parseObject(run.contextSnapshot);
|
||
const isTerminalSessionGoalControl =
|
||
runContext.resumeIntent === true &&
|
||
readNonEmptyString(runContext.goalControlRequestId) !== null;
|
||
const issueId = issueIdFromRunContext(run.contextSnapshot);
|
||
const preservesTerminalIssueReport = isTerminalSessionGoalControl ||
|
||
(issueId !== null && await isCompletedOnboardingHandoffWake(db, {
|
||
companyId: run.companyId,
|
||
issueId,
|
||
agentId: run.agentId,
|
||
reason: readNonEmptyString(runContext.wakeReason),
|
||
contextSnapshot: runContext,
|
||
}));
|
||
let issueTerminalStatus: "succeeded" | "cancelled" | null =
|
||
preservesTerminalIssueReport
|
||
? null
|
||
: (options?.referencingIssueTerminalStatus ?? null);
|
||
if (
|
||
!preservesTerminalIssueReport &&
|
||
!issueTerminalStatus &&
|
||
!options?.runReferencedByActiveIssue &&
|
||
issueId
|
||
) {
|
||
const issueStatus = await db
|
||
.select({ status: issues.status })
|
||
.from(issues)
|
||
.where(eq(issues.id, issueId))
|
||
.then((rows) => rows[0]?.status ?? null);
|
||
if (issueStatus === "done") issueTerminalStatus = "succeeded";
|
||
else if (issueStatus === "cancelled") issueTerminalStatus = "cancelled";
|
||
}
|
||
|
||
// Process-death authority. The run is live while either its adapter process
|
||
// or the enclosing heartbeat execution/finalization still owns it. Check
|
||
// that full in-process lifecycle first, then the recorded pid and process
|
||
// group. Require recorded process metadata, so this authority never fires
|
||
// on a run that has not yet stored its pid.
|
||
let processGone = false;
|
||
const hasLiveExecution =
|
||
deps.liveRunExecutions?.has(run.id) ?? runningProcesses.has(run.id);
|
||
if (!hasLiveExecution) {
|
||
if (typeof pid === "number" || typeof processGroupId === "number") {
|
||
const processAlive =
|
||
(typeof pid === "number" && isPidAlive(pid)) ||
|
||
(typeof processGroupId === "number" &&
|
||
isProcessGroupAlive(processGroupId));
|
||
processGone = !processAlive;
|
||
}
|
||
}
|
||
|
||
// Neither authority applies. The run is still live, so leave it alone.
|
||
if (!issueTerminalStatus && !processGone) {
|
||
return { terminalized: false, status: run.status };
|
||
}
|
||
|
||
const authority = issueTerminalStatus ? "issue_terminal" : "process_gone";
|
||
const terminalStatus = issueTerminalStatus ?? "interrupted";
|
||
const errorCode = issueTerminalStatus
|
||
? "orphaned_running_run_issue_terminal"
|
||
: "orphaned_running_run";
|
||
const message =
|
||
authority === "issue_terminal"
|
||
? "run terminalized by recovery backstop: issue reached a terminal status while heartbeat_runs.status stayed live"
|
||
: "run terminalized by recovery backstop: process and sandbox gone while heartbeat_runs.status stayed live";
|
||
|
||
await deps.beforeOrphanedRunTerminalWrite?.(run.id);
|
||
const now = new Date();
|
||
const updated = await db
|
||
.update(heartbeatRuns)
|
||
.set({
|
||
status: terminalStatus,
|
||
finishedAt: run.finishedAt ?? now,
|
||
error: run.error ?? (terminalStatus === "interrupted" ? message : null),
|
||
errorCode:
|
||
run.errorCode ??
|
||
(terminalStatus === "interrupted" ? errorCode : null),
|
||
updatedAt: now,
|
||
})
|
||
.where(
|
||
and(
|
||
eq(heartbeatRuns.id, run.id),
|
||
eq(heartbeatRuns.status, "running"),
|
||
eq(heartbeatRuns.runtimeMode, run.runtimeMode),
|
||
nativeRunnerOwnershipNotHeldCondition(),
|
||
// Provider exit is expected while the native coordinator resumes a
|
||
// session or copies its completed workspace back. The coordinator
|
||
// owns those retries, including expired leases and future attempts.
|
||
// Check at the write so a newly recorded result cannot be orphaned
|
||
// using the earlier liveness snapshot. Terminal issue status remains
|
||
// the stronger authority.
|
||
!issueTerminalStatus && run.runtimeMode === "native"
|
||
? notExists(db.select({ runId: nativeRunFinalizations.runId })
|
||
.from(nativeRunFinalizations).where(and(
|
||
eq(nativeRunFinalizations.runId, heartbeatRuns.id),
|
||
eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId),
|
||
or(
|
||
and(
|
||
isNotNull(nativeRunFinalizations.resultId),
|
||
inArray(nativeRunFinalizations.phase, [
|
||
"observed", "workspace_finalizing", "ready_for_assessment",
|
||
"arbitrating", "retryable_failure",
|
||
]),
|
||
),
|
||
and(
|
||
isNull(nativeRunFinalizations.resultId),
|
||
or(
|
||
eq(nativeRunFinalizations.phase, "retryable_failure"),
|
||
and(eq(nativeRunFinalizations.phase, "observed"), gt(nativeRunFinalizations.attempt, 0)),
|
||
),
|
||
),
|
||
),
|
||
)))
|
||
: undefined,
|
||
// Recheck ownership in the write: a controller can renew or claim
|
||
// the run after the liveness read. An old snapshot cannot end a new
|
||
// controller's run, even if that controller's lease later expires.
|
||
run.runtimeMode === "legacy"
|
||
? and(
|
||
run.controllerBootId
|
||
? eq(heartbeatRuns.controllerBootId, run.controllerBootId)
|
||
: isNull(heartbeatRuns.controllerBootId),
|
||
or(
|
||
isNull(heartbeatRuns.controllerBootId),
|
||
sql`${heartbeatRuns.controllerLeaseExpiresAt} <= clock_timestamp()`,
|
||
),
|
||
)
|
||
: undefined,
|
||
),
|
||
)
|
||
.returning()
|
||
.then((rows) => rows[0] ?? null);
|
||
if (!updated) {
|
||
// Another path finalized the run between the read and this write. Keep
|
||
// that terminal outcome authoritative.
|
||
const [current] = await db
|
||
.select({ status: heartbeatRuns.status })
|
||
.from(heartbeatRuns)
|
||
.where(eq(heartbeatRuns.id, run.id));
|
||
return { terminalized: false, status: current?.status ?? run.status };
|
||
}
|
||
|
||
// Telemetry is best-effort background work; it must not delay clearing
|
||
// the stale lock below, so fire it and do not await it.
|
||
void emitAgentTaskRun(db, updated);
|
||
runningProcesses.delete(run.id);
|
||
// The run update above already committed the terminal status. The audit
|
||
// event is best-effort: if the insert fails, the caller must still treat
|
||
// the run as terminalized and clear the lock in the same sweep. So catch
|
||
// the failure, log it, and continue. A thrown error here would abort the
|
||
// sweep and leave the stale lock in place.
|
||
try {
|
||
await appendRecoveryRunEvent(updated, {
|
||
level: "warn",
|
||
message,
|
||
payload: {
|
||
source: "recovery.sweep_stale_issue_locks",
|
||
authority,
|
||
previousStatus: run.status,
|
||
terminalStatus,
|
||
...(issueId ? { issueId } : {}),
|
||
pid,
|
||
processGroupId,
|
||
},
|
||
});
|
||
} catch (error) {
|
||
logger.error(
|
||
{ err: error, runId: run.id, previousStatus: run.status },
|
||
"failed to append recovery run event after terminalizing orphaned run; run stays terminal and the sweep clears the lock",
|
||
);
|
||
}
|
||
logger.warn(
|
||
{
|
||
runId: run.id,
|
||
authority,
|
||
previousStatus: run.status,
|
||
terminalStatus,
|
||
issueId,
|
||
pid,
|
||
processGroupId,
|
||
},
|
||
"terminalized orphaned running heartbeat run in stale-lock sweep",
|
||
);
|
||
return { terminalized: true, status: updated.status };
|
||
}
|
||
|
||
// Backstop sweeper: clears stale lock columns on issues whose checkoutRunId
|
||
// or executionRunId points at a heartbeat_runs row that is either missing or
|
||
// in a terminal status. Provides self-heal for stale locks that fell outside
|
||
// releaseIssueExecutionAndPromote / clearCheckoutRunIfTerminal / adoption.
|
||
// Before it evaluates cleanability, it terminalizes any referenced run that
|
||
// still claims to be live but can no longer reach a terminal status on its
|
||
// own, so a stuck "running" run can no longer block the sweep. Idempotent and
|
||
// safe: clears at most one row's worth of lock columns per candidate.
|
||
async function sweepStaleIssueLocks() {
|
||
const result = {
|
||
cleared: 0,
|
||
issueIds: [] as string[],
|
||
terminalizedRunIds: [] as string[],
|
||
};
|
||
|
||
const candidates = await db
|
||
.select({
|
||
id: issues.id,
|
||
companyId: issues.companyId,
|
||
status: issues.status,
|
||
checkoutRunId: issues.checkoutRunId,
|
||
executionRunId: issues.executionRunId,
|
||
})
|
||
.from(issues)
|
||
.where(
|
||
sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
|
||
);
|
||
|
||
const referencedRunIds = [
|
||
...new Set(
|
||
candidates
|
||
.flatMap((issue) => [issue.checkoutRunId, issue.executionRunId])
|
||
.filter((id): id is string => !!id),
|
||
),
|
||
];
|
||
const runRows =
|
||
referencedRunIds.length > 0
|
||
? await db
|
||
.select()
|
||
.from(heartbeatRuns)
|
||
.where(inArray(heartbeatRuns.id, referencedRunIds))
|
||
: [];
|
||
const runStatusById = new Map<string, string>();
|
||
for (const row of runRows) runStatusById.set(row.id, row.status);
|
||
const runById = new Map(runRows.map(row => [row.id, row]));
|
||
|
||
// Collect the runs that a non-terminal issue still references. Such a run is
|
||
// the live run of an active issue. A different, terminal issue can also hold
|
||
// the same run id in a stale lock column. The terminal reference alone must
|
||
// not terminalize a run that an active issue still owns, so exclude these
|
||
// runs from the issue-terminal authority below.
|
||
const runIdsReferencedByActiveIssue = new Set<string>();
|
||
for (const issue of candidates) {
|
||
if (issue.status === "done" || issue.status === "cancelled") continue;
|
||
for (const runId of [issue.checkoutRunId, issue.executionRunId]) {
|
||
if (runId) runIdsReferencedByActiveIssue.add(runId);
|
||
}
|
||
}
|
||
|
||
// Map each referenced run to the terminal run status implied by its
|
||
// referencing issue. When a terminal issue still holds the run in a lock
|
||
// column, that run is orphaned: the issue is the stuck "Live" task the UI
|
||
// shows. A "done" issue implies "succeeded"; a "cancelled" issue implies
|
||
// "cancelled". Skip a run that an active issue also references, because that
|
||
// run is still live for the active issue.
|
||
const issueTerminalStatusByRunId = new Map<
|
||
string,
|
||
"succeeded" | "cancelled"
|
||
>();
|
||
for (const issue of candidates) {
|
||
const implied =
|
||
issue.status === "done"
|
||
? "succeeded"
|
||
: issue.status === "cancelled"
|
||
? "cancelled"
|
||
: null;
|
||
if (!implied) continue;
|
||
for (const runId of [issue.checkoutRunId, issue.executionRunId]) {
|
||
if (runId && !runIdsReferencedByActiveIssue.has(runId)) {
|
||
issueTerminalStatusByRunId.set(runId, implied);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Pre-pass: terminalize any referenced run that still claims to be live but
|
||
// can no longer reach a terminal status on its own. This lets the sweep
|
||
// clear the lock in the same pass instead of waiting for the run to reach a
|
||
// terminal status by another route.
|
||
for (const row of runRows) {
|
||
const outcome = await terminalizeOrphanedRunningRun(row, {
|
||
referencingIssueTerminalStatus:
|
||
issueTerminalStatusByRunId.get(row.id) ?? null,
|
||
runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id),
|
||
});
|
||
runStatusById.set(row.id, outcome.status);
|
||
if (outcome.terminalized) result.terminalizedRunIds.push(row.id);
|
||
}
|
||
|
||
const isCleanable = (runId: string | null) => {
|
||
if (!runId) return true;
|
||
const status = runStatusById.get(runId);
|
||
if (!status) return true; // missing run row → no real claim
|
||
return TERMINAL_HEARTBEAT_RUN_STATUSES.has(status);
|
||
};
|
||
|
||
for (const issue of candidates) {
|
||
const originalOwner = issue.executionRunId ? runById.get(issue.executionRunId) : undefined;
|
||
// The pre-pass can lose its terminal write to the executor and observe a
|
||
// newer terminal status. Do not test claim ownership using the old status.
|
||
const owner = originalOwner ? { ...originalOwner,
|
||
status: runStatusById.get(originalOwner.id) ?? originalOwner.status } : undefined;
|
||
if (owner && isExplicitContinuationRetryClaim(issue, owner)) {
|
||
// A terminal row can still own cleanup and a pending bounded retry.
|
||
// Re-enter the same policy rather than erasing its exact-owner proof.
|
||
// That policy handles pending cleanup, newer input, and final denial.
|
||
await deps.settleExplicitContinuationRetry?.(owner);
|
||
continue;
|
||
}
|
||
if (
|
||
!isCleanable(issue.checkoutRunId) ||
|
||
!isCleanable(issue.executionRunId)
|
||
) {
|
||
continue;
|
||
}
|
||
|
||
const updated = await db
|
||
.update(issues)
|
||
.set({
|
||
checkoutRunId: null,
|
||
executionRunId: null,
|
||
executionAgentNameKey: null,
|
||
executionLockedAt: null,
|
||
updatedAt: new Date(),
|
||
})
|
||
.where(
|
||
and(
|
||
eq(issues.id, issue.id),
|
||
issue.checkoutRunId
|
||
? eq(issues.checkoutRunId, issue.checkoutRunId)
|
||
: isNull(issues.checkoutRunId),
|
||
issue.executionRunId
|
||
? eq(issues.executionRunId, issue.executionRunId)
|
||
: isNull(issues.executionRunId),
|
||
),
|
||
)
|
||
.returning({ id: issues.id })
|
||
.then((rows) => rows[0] ?? null);
|
||
|
||
if (!updated) continue;
|
||
|
||
result.cleared += 1;
|
||
result.issueIds.push(updated.id);
|
||
|
||
await logActivity(db, {
|
||
companyId: issue.companyId,
|
||
actorType: "system",
|
||
actorId: "system",
|
||
agentId: null,
|
||
runId: null,
|
||
action: "issue.stale_lock_cleared",
|
||
entityType: "issue",
|
||
entityId: updated.id,
|
||
details: {
|
||
source: "recovery.sweep_stale_issue_locks",
|
||
clearedCheckoutRunId: issue.checkoutRunId,
|
||
clearedExecutionRunId: issue.executionRunId,
|
||
referencedRunStatuses: Object.fromEntries(runStatusById),
|
||
},
|
||
});
|
||
}
|
||
|
||
if (result.cleared > 0 || result.terminalizedRunIds.length > 0) {
|
||
logger.warn(
|
||
{
|
||
cleared: result.cleared,
|
||
issueIds: result.issueIds,
|
||
terminalizedRunIds: result.terminalizedRunIds,
|
||
},
|
||
"swept stale issue lock columns",
|
||
);
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
return {
|
||
buildRunOutputSilence,
|
||
escalateStrandedRecoveryIssueInPlace,
|
||
escalateStrandedAssignedIssue,
|
||
recordWatchdogDecision,
|
||
scanSilentActiveRuns,
|
||
reconcileStrandedAssignedIssues,
|
||
reconcileLegacyContinuation,
|
||
legacyRepairDispatchBlock,
|
||
sweepStaleIssueLocks,
|
||
reconcileResolvedDependencyWakeBackstop,
|
||
readRecoveryTimerIntervalMs,
|
||
};
|
||
}
|