From 0c4bb6fa57bb7c5c3090291617e5b95d93d77a92 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 11 Oct 2026 09:27:01 -0500 Subject: [PATCH] Define the computer admission wait retry contract Keep resource waits separate from failure accounting and require recorded task-lock and non-assignee authority. Project the provider-neutral contract from the Boat core prerequisite; keep database-backed admission proof in the integration. Co-Authored-By: Paperclip --- .../run-dispatch/domain/policy.test.ts | 6 +-- .../src/modules/run-dispatch/domain/policy.ts | 5 +- .../run-dispatch/domain/wake-context.test.ts | 9 ++++ .../run-dispatch/domain/wake-context.ts | 3 ++ .../execution-recovery-attempt.test.ts | 46 ++++++++++++++++++- .../services/execution-recovery-attempt.ts | 28 +++++++++-- 6 files changed, 87 insertions(+), 10 deletions(-) diff --git a/server/src/modules/run-dispatch/domain/policy.test.ts b/server/src/modules/run-dispatch/domain/policy.test.ts index 051c676d2c..38e0dc3b3e 100644 --- a/server/src/modules/run-dispatch/domain/policy.test.ts +++ b/server/src/modules/run-dispatch/domain/policy.test.ts @@ -510,13 +510,13 @@ describe("native replacement execution authority", () => { }); -describe("AI subscription wait ownership", () => { +describe.each(["ai_connection_wait", "computer_admission_wait"] as const)("%s ownership", (retryReasonKind) => { it.each([null, "another-run"])("preserves assignee lock guards and admits only recorded non-assignee waits: %s", (issueExecutionRunId) => { const gate = { - ...baseGateFacts(), retryReasonKind: "ai_connection_wait" as const, + ...baseGateFacts(), retryReasonKind, enforceIssueExecutionLock: true, issueExecutionRunId, }; - const queued = { ...baseStalenessFacts(), retryReasonKind: "ai_connection_wait" as const, issueExecutionRunId }; + const queued = { ...baseStalenessFacts(), retryReasonKind, issueExecutionRunId }; expect(decideScheduledRetryGate(gate, NOW)).toMatchObject({ allowed: false, errorCode: "issue_execution_lock_changed" }); expect(decideQueuedRunStaleness(queued, NOW)).toMatchObject({ stale: true, errorCode: "issue_execution_lock_changed" }); const nonAssignee = { issueAssigneeAgentId: "another-agent", isNonAssigneeWorkspaceBusyRetry: true }; diff --git a/server/src/modules/run-dispatch/domain/policy.ts b/server/src/modules/run-dispatch/domain/policy.ts index ec4b729839..a6ac7a5a5f 100644 --- a/server/src/modules/run-dispatch/domain/policy.ts +++ b/server/src/modules/run-dispatch/domain/policy.ts @@ -10,6 +10,7 @@ export type RetryReasonKind = | "max_turn_continuation" | "disposition_repair" | "native_safe_replacement" + | "computer_admission_wait" | "ai_connection_wait" | "other"; @@ -408,7 +409,7 @@ export function decideScheduledRetryGate( const lockOutcome = decideExecutionLock({ requiresExecutionLock: (requiresInProgress || - (facts.retryReasonKind === "ai_connection_wait" && !facts.isNonAssigneeWorkspaceBusyRetry)) && + ((facts.retryReasonKind === "ai_connection_wait" || facts.retryReasonKind === "computer_admission_wait") && !facts.isNonAssigneeWorkspaceBusyRetry)) && facts.enforceIssueExecutionLock, runId: facts.runId, issueExecutionRunId: facts.issueExecutionRunId, @@ -640,7 +641,7 @@ export function decideQueuedRunStaleness( const lockOutcome = decideExecutionLock({ // A server-recorded non-assignee wake never held the task execution lock. requiresExecutionLock: requiresInProgress || - (facts.retryReasonKind === "ai_connection_wait" && !facts.isNonAssigneeWorkspaceBusyRetry), + ((facts.retryReasonKind === "ai_connection_wait" || facts.retryReasonKind === "computer_admission_wait") && !facts.isNonAssigneeWorkspaceBusyRetry), runId: facts.runId, issueExecutionRunId: facts.issueExecutionRunId, }); diff --git a/server/src/modules/run-dispatch/domain/wake-context.test.ts b/server/src/modules/run-dispatch/domain/wake-context.test.ts index ac6b84eb68..83ae2dcc79 100644 --- a/server/src/modules/run-dispatch/domain/wake-context.test.ts +++ b/server/src/modules/run-dispatch/domain/wake-context.test.ts @@ -9,6 +9,15 @@ import { } from "./wake-context.js"; describe("wake context", () => { + it("requires the computer-specific non-assignee receipt for a computer wait", () => { + expect(isNonAssigneeWorkspaceBusyRetry("computer_admission_wait", { + computerAdmissionDeferredWhileAssignee: false, + })).toBe(true); + for (const context of [{}, { computerAdmissionDeferredWhileAssignee: true }, { workspaceBusyDeferredWhileAssignee: false }, { aiConnectionBusyDeferredWhileAssignee: false }]) { + expect(isNonAssigneeWorkspaceBusyRetry("computer_admission_wait", context)).toBe(false); + } + expect(isNonAssigneeWorkspaceBusyRetry("transient_failure", { computerAdmissionDeferredWhileAssignee: false })).toBe(false); + }); it("requires the subscription-specific non-assignee receipt for a subscription wait", () => { expect(isNonAssigneeWorkspaceBusyRetry("ai_connection_busy", { aiConnectionBusyDeferredWhileAssignee: false, diff --git a/server/src/modules/run-dispatch/domain/wake-context.ts b/server/src/modules/run-dispatch/domain/wake-context.ts index 2100b357d9..2516e7a2eb 100644 --- a/server/src/modules/run-dispatch/domain/wake-context.ts +++ b/server/src/modules/run-dispatch/domain/wake-context.ts @@ -16,6 +16,7 @@ function readNonEmptyString(value: unknown): string | null { } export const MAX_TURN_CONTINUATION_RETRY_REASON = "max_turns_continuation"; +export const COMPUTER_ADMISSION_WAIT_RETRY_REASON = "computer_admission_wait"; export const WORKSPACE_BUSY_RETRY_REASON = "workspace_busy"; export const AI_CONNECTION_POOL_WAIT_RETRY_REASON = "ai_connection_pool_wait"; export const AI_CONNECTION_BUSY_RETRY_REASON = "ai_connection_busy"; @@ -40,6 +41,8 @@ export function isNonAssigneeWorkspaceBusyRetry( contextSnapshot: Record, ): boolean { return ( + (retryReason === COMPUTER_ADMISSION_WAIT_RETRY_REASON && + contextSnapshot.computerAdmissionDeferredWhileAssignee === false) || (retryReason === WORKSPACE_BUSY_RETRY_REASON && contextSnapshot.workspaceBusyDeferredWhileAssignee === false) || ((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) && diff --git a/server/src/services/execution-recovery-attempt.test.ts b/server/src/services/execution-recovery-attempt.test.ts index c434e7b60a..b3e2778936 100644 --- a/server/src/services/execution-recovery-attempt.test.ts +++ b/server/src/services/execution-recovery-attempt.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; +import { executionFailureRetryCount, isComputerAdmissionWait } from "./execution-recovery-attempt.js"; describe("failure attempts across resource waits", () => { it("preserves prior failures through subscription waits without trusting unrelated context", () => { @@ -27,7 +27,7 @@ describe("failure attempts across resource waits", () => { }); describe("persisted independent accounting", () => { - it.each(["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"])("%s cannot erase prior infrastructure debits or spend more", scheduledRetryReason => { + it.each(["computer_admission_wait", "max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"])("%s cannot erase prior infrastructure debits or spend more", scheduledRetryReason => { expect(executionFailureRetryCount({ scheduledRetryReason, scheduledRetryAttempt: 20, contextSnapshot: { executionRetryAccounting: { version: 1, failureRetries: 2, maxTurnContinuations: 1 } }, })).toBe(2); @@ -43,3 +43,45 @@ describe("persisted independent accounting", () => { })).toBe(2); }); }); + + +describe("structural computer admission wait receipt", () => { + function receipt() { + return { + id: "run-1", companyId: "company-1", status: "cancelled", runtimeMode: "legacy", + errorCode: "computer_admission_wait", + resultJson: { + executionRecovery: { kind: "computer_admission_wait", providerWorkStarted: false }, + cancellation: { expected: true, source: "control_plane", initiator: { type: "system" } }, + computerAdmission: { + runId: "run-1", companyId: "company-1", + computerId: "11111111-1111-4111-8111-111111111111", + environmentId: "22222222-2222-4222-8222-222222222222", stopId: "stop-1", + }, + }, + }; + } + + it("recognizes the recorded never-dispatched wait without treating a code alone as proof", () => { + const run = receipt(); + expect(isComputerAdmissionWait(run)).toBe(true); + expect(isComputerAdmissionWait({ ...run, resultJson: {} })).toBe(false); + expect(isComputerAdmissionWait({ ...run, resultJson: { ...run.resultJson, executionRecovery: { kind: "computer_admission_wait", providerWorkStarted: true } } })).toBe(false); + expect(isComputerAdmissionWait({ ...run, resultJson: { ...run.resultJson, cancellation: { ...run.resultJson.cancellation, initiator: { type: "user" } } } })).toBe(false); + }); + + it("rejects foreign run/company receipts, invalid identities and already settled retries", () => { + const run = receipt(); + for (const computerAdmission of [ + { ...run.resultJson.computerAdmission, runId: "another-run" }, + { ...run.resultJson.computerAdmission, companyId: "another-company" }, + { ...run.resultJson.computerAdmission, computerId: "invalid" }, + { ...run.resultJson.computerAdmission, environmentId: "invalid" }, + { ...run.resultJson.computerAdmission, stopId: "" }, + ]) expect(isComputerAdmissionWait({ ...run, resultJson: { ...run.resultJson, computerAdmission } })).toBe(false); + expect(isComputerAdmissionWait({ ...run, resultJson: { ...run.resultJson, computerAdmissionRetryOutcome: "superseded" } })).toBe(false); + expect(isComputerAdmissionWait({ ...run, runnerProfileJson: { nativeExecutionInput: { version: 1 } } })).toBe(false); + expect(isComputerAdmissionWait({ ...run, runtimeMode: "native" })).toBe(false); + expect(isComputerAdmissionWait({ ...run, status: "failed" })).toBe(false); + }); +}); diff --git a/server/src/services/execution-recovery-attempt.ts b/server/src/services/execution-recovery-attempt.ts index dec0d847cf..85d8ef1063 100644 --- a/server/src/services/execution-recovery-attempt.ts +++ b/server/src/services/execution-recovery-attempt.ts @@ -32,6 +32,10 @@ function historicalFailureCount(run: RetryRun): number { const saved = count(run.contextSnapshot?.failureRetriesBeforeAiConnectionWait); if (saved !== null) return saved; } + if (run.scheduledRetryReason === "computer_admission_wait") { + const saved = count(run.contextSnapshot?.failureRetriesBeforeComputerWait); + if (saved !== null) return saved; + } if (run.scheduledRetryReason === "workspace_busy") { const saved = count(run.contextSnapshot?.failureRetriesBeforeWorkspaceWait); if (saved !== null) return saved; @@ -42,7 +46,7 @@ function historicalFailureCount(run: RetryRun): number { export function executionRetryAccounting(run: RetryRun): ExecutionRetryAccounting { const saved = savedAccounting(run); - const nonFailureLane = ["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"].includes(run.scheduledRetryReason ?? ""); + const nonFailureLane = ["computer_admission_wait", "max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"].includes(run.scheduledRetryReason ?? ""); return { version: 1, failureRetries: Math.max(saved?.failureRetries ?? 0, saved && nonFailureLane ? 0 : historicalFailureCount(run)), @@ -57,7 +61,7 @@ export function executionFailureRetryCount(run: RetryRun): number { } export function executionRetryAttemptCount(run: RetryRun, reason: string): number { - if (reason === "workspace_busy" || reason === "ai_connection_busy" || reason === "ai_connection_pool_wait") { + if (reason === "computer_admission_wait" || reason === "workspace_busy" || reason === "ai_connection_busy" || reason === "ai_connection_pool_wait") { return run.scheduledRetryReason === reason ? count(run.scheduledRetryAttempt) ?? 0 : 0; } const accounting = executionRetryAccounting(run); @@ -67,6 +71,24 @@ export function executionRetryAttemptCount(run: RetryRun, reason: string): numbe export function accountingForScheduledRetry(run: RetryRun, reason: string, attempt: number): ExecutionRetryAccounting { const accounting = executionRetryAccounting(run); if (reason === "max_turns_continuation") accounting.maxTurnContinuations = attempt; - else if (reason !== "workspace_busy" && reason !== "ai_connection_busy" && reason !== "ai_connection_pool_wait") accounting.failureRetries = attempt; + else if (reason !== "computer_admission_wait" && reason !== "workspace_busy" && reason !== "ai_connection_busy" && reason !== "ai_connection_pool_wait") accounting.failureRetries = attempt; return accounting; } + +/** Only the executor can attest that a pending-stop wait never dispatched and finished cleanup. */ +export function isComputerAdmissionWait(run: { + id: string; companyId: string; status: string; runtimeMode: string; errorCode?: string | null; + resultJson?: Record | null; runnerProfileJson?: Record | null; +}): boolean { + const object = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + const result = object(run.resultJson), recovery = object(result.executionRecovery), cancellation = object(result.cancellation), admission = object(result.computerAdmission); + return run.runtimeMode === "legacy" && run.status === "cancelled" && run.errorCode === "computer_admission_wait" && + recovery.kind === "computer_admission_wait" && recovery.providerWorkStarted === false && + result.computerAdmissionRetryOutcome == null && + cancellation.expected === true && cancellation.source === "control_plane" && + object(cancellation.initiator).type === "system" && + admission.runId === run.id && admission.companyId === run.companyId && + [admission.computerId, admission.environmentId].every(value => typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) && + typeof admission.stopId === "string" && admission.stopId.length > 0 && + !object(run.runnerProfileJson).nativeExecutionInput; +}