Files
PaperClipAI/server/src/services/native-runtime/status-arbiter.ts
T
Devin FoleyandPaperclip 332e4c5df9 fix(native): settle completed results that retain a valid monitor (#17641)
Preserve a valid saved monitor after its successful source releases execution and checkout. Recheck accepted result identity, receipts, ownership and governance under the task lock. Notify the existing scheduler once and preserve accepted live and recovered replies.

Apex 5/5 after fixing its reply-presentation finding. All CI checks, 191 focused tests, 293 broader native tests, workspace typecheck and server build pass. Hosted effects require staging verification.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-25 10:47:02 -07:00

662 lines
25 KiB
TypeScript

import type { NativeEvidenceAssessment } from "./evidence-classifier.js";
import { NATIVE_MODEL_REJECTION_MESSAGE, NATIVE_PROVIDER_CAPACITY_MAX_RETRIES, NATIVE_PROVIDER_OVERLOADED_CODE, NATIVE_PROVIDER_OVERLOADED_MESSAGE } from "./native-provider-failure.js";
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v13";
export type NativeAuthoritativeIssueStatus =
| "backlog"
| "todo"
| "in_progress"
| "in_review"
| "blocked"
| "done"
| "cancelled";
export type NativeGovernanceGate = {
kind: "approval" | "interaction" | "execution_stage";
id: string;
};
export type NativeStatusEffect =
| { kind: "create_interaction"; gate?: NativeGovernanceGate; prompt?: string }
| {
kind: "bind_reviewer";
requestKey?: string;
prompt: string;
detailsMarkdown?: string | null;
ownerUserId?: string | null;
ownerAgentId?: string | null;
}
| { kind: "notify_owner"; agentId: string; reason: string }
| {
kind: "enqueue_continuation";
continuationKind:
| "same_agent"
| "retry"
| "delegated_issue"
| "response_wake"
| "monitor";
summary: string;
idempotencyKey: string;
agentId: string;
}
| {
kind: "bind_blocker";
owner: { agentId: string } | "board";
action: string;
}
| { kind: "schedule_retry"; cause: string; summary: string; agentId: string }
| {
kind: "record_finalization_error";
cause: string;
nextAction: string;
agentId: string;
}
| { kind: "release_run_resources" }
| { kind: "create_delegated_issue"; agentId: string; summary: string }
| { kind: "accept_replacement_turn" }
| { kind: "cancel_continuations" }
| { kind: "append_superseding_assessment" }
| { kind: "dispatch_pending_effect" }
| { kind: "increment_status_version" }
| { kind: "schedule_reconciliation" }
| { kind: "record_shadow_decision" }
| { kind: "render_four_layers" }
| { kind: "materialize_contract" }
| { kind: "record_mode_labeled_divergence" }
| { kind: "record_mode_native" }
| { kind: "record_policy_version" }
| { kind: "finish_as_native" }
| { kind: "resume_workspace_operation" }
| { kind: "record_expiry" }
| { kind: "record_stale_response" }
| { kind: "link_canonical_request" }
| {
kind: "record_recovery";
cause: string;
nextAction: string;
agentId: string;
}
| { kind: "release_checkout" };
export interface NativeStatusDecision {
policyVersion: typeof NATIVE_STATUS_ARBITER_POLICY_VERSION;
statusAction: NativeAuthoritativeIssueStatus | "preserve";
toStatus: NativeAuthoritativeIssueStatus;
reasonCode: string | null;
unblockDescriptor: {
owner: { agentId: string } | "board";
action: string;
} | null;
effects: NativeStatusEffect[];
}
/** Pure authority boundary: model prose is evidence, never a status command. */
export function arbitrateNativeStatus(input: {
assessment: NativeEvidenceAssessment;
terminalState: "succeeded" | "failed" | "cancelled";
workspaceFinalizeStatus: "succeeded" | "failed";
governanceGate?: NativeGovernanceGate | null;
completionClaimPolicyAccepted?: boolean;
allowIncompleteContinuation?: boolean;
/**
* True when the issue already has a durable unresolved dependency edge.
* A "current_track" result cannot remain in_progress in that state because
* checkout is intentionally gated until the dependency resolves.
*/
hasUnresolvedIssueBlockers?: boolean;
/** A newer native child result still has an undelivered parent wake. */
hasPendingChildCompletion?: boolean;
/** A governance interaction created by this run was accepted before the run settled. */
governanceResolvedForRun?: boolean;
externalChatResponseWaitAuthorization?:
"authorized" | "revoked" | "not_applicable";
publishedExternalReview?: boolean;
/** Server-verified accepted plan request and normal provider terminal. */
planWaitAuthorized?: boolean;
monitorWaitAuthorized?: boolean;
caseHandoffAuthorized?: boolean;
/** The saved timer already admitted another run and this run no longer owns checkout. */
monitorWaitDispatched?: boolean;
monitorWaitPreserved?: boolean;
boardResponseWaitAuthorized?: boolean;
boardResponseWaitOrigin?: boolean;
isConversation?: boolean;
hasActivePauseHold?: boolean;
reviewOwnerUserId?: string | null;
/** Review decisions own task state; a reviewer's finish report cannot override them. */
nativeReviewOutcome?: "resolved" | "pending" | "stale";
/** Re-derived from the committed runner terminal and pinned execution identity. */
providerModelRejected?: boolean;
providerOverloaded?: boolean;
providerFailureSuperseded?: boolean;
failureRetryCount?: number;
agentId: string;
priorIssueStatus: NativeAuthoritativeIssueStatus;
}): NativeStatusDecision {
if (["done", "cancelled"].includes(input.priorIssueStatus)) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "terminal_status_preserved",
unblockDescriptor: null,
effects: [],
};
}
if (input.workspaceFinalizeStatus !== "succeeded") {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "finalization_failed_claim_preserved",
unblockDescriptor: null,
effects: [
{
kind: "record_finalization_error",
cause: "workspace_finalization_failed",
nextAction:
"Repair and re-run workspace finalization for the persisted native result.",
agentId: input.agentId,
},
],
};
}
if (input.providerFailureSuperseded) return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve", toStatus: input.priorIssueStatus,
reasonCode: "provider_failure_authority_superseded", unblockDescriptor: null, effects: [],
};
if (input.caseHandoffAuthorized) return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve", toStatus: input.priorIssueStatus,
reasonCode: "case_handoff_waiting_for_release", unblockDescriptor: null,
effects: [{ kind: "release_checkout" }],
};
if (input.providerModelRejected && input.terminalState === "failed" &&
input.nativeReviewOutcome !== "stale" && input.nativeReviewOutcome !== "resolved") {
if (input.nativeReviewOutcome === "pending") {
// The pending review is bound to the worker's status decision and
// version. Preserve that authority so an explicit retry after a
// configuration repair can resolve the same review. In particular,
// do not create the normal unresolved-review recovery action.
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "native_provider_model_rejected",
unblockDescriptor: null,
effects: [{ kind: "release_checkout" }],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "blocked",
toStatus: "blocked",
reasonCode: "native_provider_model_rejected",
unblockDescriptor: { owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE },
effects: [{ kind: "bind_blocker", owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }],
};
}
if (input.providerOverloaded && input.terminalState === "failed" &&
input.nativeReviewOutcome !== "stale" && input.nativeReviewOutcome !== "resolved") {
const held = input.governanceGate || input.hasUnresolvedIssueBlockers || input.hasActivePauseHold || input.priorIssueStatus === "blocked";
const exhausted = (input.failureRetryCount ?? 0) >= NATIVE_PROVIDER_CAPACITY_MAX_RETRIES;
if (held || (exhausted && input.nativeReviewOutcome === "pending")) return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve", toStatus: input.priorIssueStatus,
reasonCode: held ? "native_provider_overloaded_held" : "native_provider_overloaded_exhausted",
unblockDescriptor: null, effects: [{ kind: "release_checkout" }],
};
if (exhausted) {
const action = `${NATIVE_PROVIDER_OVERLOADED_MESSAGE} Automatic retries exhausted; retry later or choose a different model.`;
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "blocked", toStatus: "blocked", reasonCode: "native_provider_overloaded_exhausted",
unblockDescriptor: { owner: "board", action },
effects: [{ kind: "bind_blocker", owner: "board", action }],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve", toStatus: input.priorIssueStatus,
reasonCode: NATIVE_PROVIDER_OVERLOADED_CODE, unblockDescriptor: null,
effects: [
{ kind: "schedule_retry", cause: NATIVE_PROVIDER_OVERLOADED_CODE, summary: "Retry after model capacity becomes available.", agentId: input.agentId },
...(input.nativeReviewOutcome === "pending" ? [{ kind: "release_checkout" as const }] : []),
],
};
}
if (input.nativeReviewOutcome) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve", toStatus: input.priorIssueStatus,
reasonCode: input.nativeReviewOutcome === "pending" ? "native_review_unresolved" : "native_review_action_finished",
unblockDescriptor: null,
effects: [
{ kind: "release_checkout" },
...(input.nativeReviewOutcome === "pending" ? [{
kind: "record_recovery" as const, cause: "native_review_unresolved",
nextAction: "The named reviewer must resolve the pending completion review or explain the concrete blocker. Do not restart the worker's assignment or create another review request.",
agentId: input.agentId,
}] : []),
],
};
}
if (input.terminalState !== "succeeded") {
if (input.terminalState === "cancelled") {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "cancellation_run_only",
unblockDescriptor: null,
effects: [{ kind: "release_run_resources" }],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "run_failed_partial_evidence_preserved",
unblockDescriptor: null,
effects: [
{
kind: "schedule_retry",
cause: "native_run_failed",
summary:
"Resume the persisted native run without opening a second provider session.",
agentId: input.agentId,
},
],
};
}
if (input.governanceGate) {
if (
input.assessment.reportedDisposition === "yielded" &&
input.assessment.continuation?.kind === "response_wake"
) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_review",
toStatus: "in_review",
reasonCode: "governed_response_waiting",
unblockDescriptor: null,
// The existing interaction is the liveness path. Its authoritative
// resolution owns the response wake; dispatching one now would start a
// duplicate provider turn before the human has answered.
effects: [{ kind: "create_interaction", gate: input.governanceGate }],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_review",
toStatus: "in_review",
reasonCode: "governed_gate_pending",
unblockDescriptor: null,
effects: [
{ kind: "create_interaction", gate: input.governanceGate },
{
kind: "notify_owner",
agentId: input.agentId,
reason: "governed_gate_pending",
},
],
};
}
if (
input.governanceResolvedForRun === true &&
["needs_review", "blocked"].includes(
input.assessment.reportedDisposition,
) &&
input.assessment.attentionRequests.length > 0 &&
input.assessment.attentionRequests.every(
(request) =>
request.ownerClass === "human" &&
["review", "approval"].includes(request.kind),
)
) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_review",
toStatus: "in_review",
reasonCode: "governance_response_continuation_queued",
unblockDescriptor: null,
// The accepted interaction already queued the response wake. Creating a
// second completion-review interaction here would strand that continuation.
// Planning turns commonly report `blocked` while awaiting their own
// confirmation; once accepted, that response wake is authoritative too.
effects: [],
};
}
const unfinishedResponseWait =
!input.isConversation &&
!["authorized", "revoked"].includes(input.externalChatResponseWaitAuthorization ?? "") &&
input.assessment.reportedDisposition === "yielded" &&
input.assessment.continuation?.kind === "response_wake" &&
input.assessment.hasBlockingRemainingWork;
if (
input.hasUnresolvedIssueBlockers === true &&
(["done", "blocked"].includes(input.assessment.reportedDisposition) || unfinishedResponseWait || input.assessment.continuation?.kind === "monitor")
) {
const owner = input.assessment.blocker?.boardOwned
? ("board" as const)
: { agentId: input.agentId };
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "blocked",
toStatus: "blocked",
reasonCode: "durable_dependency_blocker_bound",
unblockDescriptor: {
owner,
action:
input.assessment.blocker?.unblockAction ??
"Wait for the issue's durable dependency blockers to complete.",
},
// Dependency completion owns the wake. Do not immediately rerun the
// blocked issue merely because the provider mislabeled its scope/status.
effects: [],
};
}
const evidenceComplete =
input.assessment.reportedDisposition === "done" &&
input.assessment.objectiveSatisfied &&
input.assessment.allCriteriaSatisfied &&
input.assessment.verificationPassed &&
!input.assessment.hasFailedVerification &&
input.assessment.attentionRequests.length === 0 &&
!input.assessment.hasBlockingRemainingWork;
const policyClaimComplete =
input.completionClaimPolicyAccepted === true &&
input.assessment.reportedDisposition === "done" &&
input.assessment.contractRevisionMatches &&
input.assessment.objectiveClaimSatisfied &&
input.assessment.criterionAssessments.length > 0 &&
input.assessment.criterionAssessments.every(
(entry) => entry.claimStatus === "satisfied",
) &&
!input.assessment.hasFailedVerification &&
input.assessment.attentionRequests.length === 0 &&
!input.assessment.hasBlockingRemainingWork;
const complete = evidenceComplete || policyClaimComplete;
if (complete && input.hasPendingChildCompletion) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_progress",
toStatus: "in_progress",
reasonCode: "native_child_completion_pending",
unblockDescriptor: null,
// The durable outbox/deferred wake already owns exactly one continuation.
effects: [{ kind: "release_checkout" }],
};
}
if (complete) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "done",
toStatus: "done",
reasonCode: input.completionClaimPolicyAccepted
? "completion_claim_policy_accepted"
: "completion_contract_satisfied",
unblockDescriptor: null,
effects: [{ kind: "release_checkout" }],
};
}
// A completion claim is not a request for human approval. Only a concrete,
// explicitly reported attention request may create a review interaction.
if (input.assessment.attentionRequests.length > 0) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_review",
toStatus: "in_review",
reasonCode: "actionable_attention_pending",
unblockDescriptor: null,
effects: input.assessment.attentionRequests.map((request, index) => ({
kind: "bind_reviewer",
requestKey: `attention-${index}`,
prompt: request.summary.slice(0, 1_000),
detailsMarkdown: input.assessment.summary,
ownerUserId: request.ownerClass === "agent" ? null : (input.reviewOwnerUserId ?? null),
ownerAgentId: request.ownerClass === "agent" ? request.targetAgentId : null,
})),
};
}
if (
input.assessment.reportedDisposition === "blocked" &&
input.assessment.blocker
) {
const owner = input.assessment.blocker.boardOwned
? ("board" as const)
: { agentId: input.agentId };
if (input.assessment.blocker.scope === "task_wide") {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "blocked",
toStatus: "blocked",
reasonCode: "task_wide_blocker_bound",
unblockDescriptor: {
owner,
action: input.assessment.blocker.unblockAction,
},
effects: [
{
kind: "bind_blocker",
owner,
action: input.assessment.blocker.unblockAction,
},
{
kind: "notify_owner",
agentId: input.agentId,
reason: "task_wide_blocker_bound",
},
],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "blocked",
toStatus: "blocked",
reasonCode: "current_track_blocker_waiting",
// A provider's current_track label is not evidence that another
// authorized, productive track exists. Bind the actual unblock request
// without waking the same agent to repeat the blocked work or old title.
unblockDescriptor: {
owner: "board",
action: input.assessment.blocker.unblockAction,
},
effects: [
{
kind: "bind_blocker",
owner: "board",
action: input.assessment.blocker.unblockAction,
},
],
};
}
if (input.assessment.reportedDisposition === "yielded" && input.assessment.continuation?.kind === "monitor") {
if (input.monitorWaitDispatched) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve", toStatus: input.priorIssueStatus,
reasonCode: "scheduled_monitor_dispatched", unblockDescriptor: null,
// Delivery already happened. Preserve successor ownership and never
// re-arm its timer or repeat the completed provider turn.
effects: [],
};
}
if (input.monitorWaitPreserved) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve", toStatus: input.priorIssueStatus,
reasonCode: "scheduled_monitor_preserved", unblockDescriptor: null, effects: [],
};
}
if (input.monitorWaitAuthorized) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: input.publishedExternalReview ? "in_review" : "preserve",
toStatus: input.publishedExternalReview ? "in_review" : input.priorIssueStatus,
reasonCode: "scheduled_monitor_waiting", unblockDescriptor: null,
// The persisted timer owns delivery, never enqueue an immediate wake.
effects: [{ kind: "release_checkout" }],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve", toStatus: input.priorIssueStatus,
reasonCode: "monitor_wait_authority_lost", unblockDescriptor: null,
effects: [{ kind: "record_finalization_error", cause: "monitor_wait_authority_lost",
nextAction: "The scheduled monitor was cleared or became ineligible. Re-establish a valid task action path.",
agentId: input.agentId }],
};
}
if (
input.assessment.reportedDisposition === "yielded" &&
input.assessment.continuation?.kind === "response_wake" &&
input.externalChatResponseWaitAuthorization === "authorized"
) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: input.publishedExternalReview ? "in_review" : "in_progress",
toStatus: input.publishedExternalReview ? "in_review" : "in_progress",
reasonCode: "external_chat_response_waiting",
unblockDescriptor: null,
// The active, authorized provider conversation is the durable liveness
// path. Only its next admitted message may enqueue the response wake.
effects: [],
};
}
if (
input.assessment.reportedDisposition === "yielded" &&
input.assessment.continuation?.kind === "response_wake" &&
input.externalChatResponseWaitAuthorization === "revoked"
) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "external_chat_response_wait_authorization_lost",
unblockDescriptor: null,
// Revocation cannot be converted into an unrequested background run.
effects: [],
};
}
if (unfinishedResponseWait && input.hasActivePauseHold) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "response_wait_pause_preserved",
unblockDescriptor: null,
effects: [],
};
}
if (
input.assessment.reportedDisposition === "yielded" &&
input.assessment.continuation?.kind === "response_wake" &&
input.boardResponseWaitAuthorized === true &&
!unfinishedResponseWait
) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_progress",
toStatus: "in_progress",
reasonCode: "board_response_waiting",
unblockDescriptor: null,
// The committer proves the current Board cause again before persisting
// the wait. Only a new user cause may wake this completed response.
effects: [],
};
}
if (
input.assessment.reportedDisposition === "yielded" &&
input.assessment.continuation?.kind === "response_wake" &&
input.boardResponseWaitOrigin &&
!input.boardResponseWaitAuthorized
) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "board_response_wait_superseded",
unblockDescriptor: null,
effects: [],
};
}
if (
input.planWaitAuthorized === true &&
input.assessment.reportedDisposition === "yielded" &&
input.assessment.continuation?.kind === "response_wake"
) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_progress", toStatus: "in_progress",
reasonCode: "native_plan_accepted_waiting_for_continuation",
unblockDescriptor: null,
// An answered native plan is not task completion or permission to
// change modes. Only a new explicit user cause may continue the work.
effects: [],
};
}
if (
input.assessment.reportedDisposition === "yielded" &&
input.assessment.continuation &&
input.assessment.continuation.kind !== "case_handoff" &&
!unfinishedResponseWait
) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_progress",
toStatus: "in_progress",
reasonCode: "live_continuation_registered",
unblockDescriptor: null,
effects: [
{
kind: "enqueue_continuation",
continuationKind: input.assessment.continuation.kind,
summary: input.assessment.continuation.summary,
idempotencyKey: input.assessment.continuation.idempotencyKey,
agentId: input.agentId,
},
],
};
}
// A current response that admits blocking work must use the server's bounded
// repair path, not a passive wait or a model-chosen continuation retry key.
if (input.allowIncompleteContinuation === false) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "prior_status_preserved_no_live_path",
unblockDescriptor: null,
effects: [
{
kind: "record_finalization_error",
cause: "completion_evidence_incomplete",
nextAction:
"Bind a durable continuation or a named recovery owner before changing issue status.",
agentId: input.agentId,
},
],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_progress",
toStatus: "in_progress",
reasonCode: "completion_evidence_incomplete",
unblockDescriptor: null,
effects: [
{
kind: "enqueue_continuation",
continuationKind: "same_agent",
summary:
"Finish the remaining work and report done, or explicitly request a named reviewer decision. Waiting for checks or an incomplete completion report does not require human approval.",
idempotencyKey: "native-completion-incomplete",
agentId: input.agentId,
},
],
};
}