mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 00:05:49 +02:00
Define the computer admission wait retry contract
Keep resource waits separate from failure accounting and require recorded task-lock and non-assignee authority. Project the provider-neutral contract from the Boat core prerequisite; keep database-backed admission proof in the integration. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
b64d630969
commit
0c4bb6fa57
6 files changed
+87
-10
No files matched your search
@@ -510,13 +510,13 @@ describe("native replacement execution authority", () => {
|
||||
});
|
||||
|
||||
|
||||
describe("AI subscription wait ownership", () => {
|
||||
describe.each(["ai_connection_wait", "computer_admission_wait"] as const)("%s ownership", (retryReasonKind) => {
|
||||
it.each([null, "another-run"])("preserves assignee lock guards and admits only recorded non-assignee waits: %s", (issueExecutionRunId) => {
|
||||
const gate = {
|
||||
...baseGateFacts(), retryReasonKind: "ai_connection_wait" as const,
|
||||
...baseGateFacts(), retryReasonKind,
|
||||
enforceIssueExecutionLock: true, issueExecutionRunId,
|
||||
};
|
||||
const queued = { ...baseStalenessFacts(), retryReasonKind: "ai_connection_wait" as const, issueExecutionRunId };
|
||||
const queued = { ...baseStalenessFacts(), retryReasonKind, issueExecutionRunId };
|
||||
expect(decideScheduledRetryGate(gate, NOW)).toMatchObject({ allowed: false, errorCode: "issue_execution_lock_changed" });
|
||||
expect(decideQueuedRunStaleness(queued, NOW)).toMatchObject({ stale: true, errorCode: "issue_execution_lock_changed" });
|
||||
const nonAssignee = { issueAssigneeAgentId: "another-agent", isNonAssigneeWorkspaceBusyRetry: true };
|
||||
|
||||
@@ -10,6 +10,7 @@ export type RetryReasonKind =
|
||||
| "max_turn_continuation"
|
||||
| "disposition_repair"
|
||||
| "native_safe_replacement"
|
||||
| "computer_admission_wait"
|
||||
| "ai_connection_wait"
|
||||
| "other";
|
||||
|
||||
@@ -408,7 +409,7 @@ export function decideScheduledRetryGate(
|
||||
const lockOutcome = decideExecutionLock({
|
||||
requiresExecutionLock:
|
||||
(requiresInProgress ||
|
||||
(facts.retryReasonKind === "ai_connection_wait" && !facts.isNonAssigneeWorkspaceBusyRetry)) &&
|
||||
((facts.retryReasonKind === "ai_connection_wait" || facts.retryReasonKind === "computer_admission_wait") && !facts.isNonAssigneeWorkspaceBusyRetry)) &&
|
||||
facts.enforceIssueExecutionLock,
|
||||
runId: facts.runId,
|
||||
issueExecutionRunId: facts.issueExecutionRunId,
|
||||
@@ -640,7 +641,7 @@ export function decideQueuedRunStaleness(
|
||||
const lockOutcome = decideExecutionLock({
|
||||
// A server-recorded non-assignee wake never held the task execution lock.
|
||||
requiresExecutionLock: requiresInProgress ||
|
||||
(facts.retryReasonKind === "ai_connection_wait" && !facts.isNonAssigneeWorkspaceBusyRetry),
|
||||
((facts.retryReasonKind === "ai_connection_wait" || facts.retryReasonKind === "computer_admission_wait") && !facts.isNonAssigneeWorkspaceBusyRetry),
|
||||
runId: facts.runId,
|
||||
issueExecutionRunId: facts.issueExecutionRunId,
|
||||
});
|
||||
|
||||
@@ -9,6 +9,15 @@ import {
|
||||
} from "./wake-context.js";
|
||||
|
||||
describe("wake context", () => {
|
||||
it("requires the computer-specific non-assignee receipt for a computer wait", () => {
|
||||
expect(isNonAssigneeWorkspaceBusyRetry("computer_admission_wait", {
|
||||
computerAdmissionDeferredWhileAssignee: false,
|
||||
})).toBe(true);
|
||||
for (const context of [{}, { computerAdmissionDeferredWhileAssignee: true }, { workspaceBusyDeferredWhileAssignee: false }, { aiConnectionBusyDeferredWhileAssignee: false }]) {
|
||||
expect(isNonAssigneeWorkspaceBusyRetry("computer_admission_wait", context)).toBe(false);
|
||||
}
|
||||
expect(isNonAssigneeWorkspaceBusyRetry("transient_failure", { computerAdmissionDeferredWhileAssignee: false })).toBe(false);
|
||||
});
|
||||
it("requires the subscription-specific non-assignee receipt for a subscription wait", () => {
|
||||
expect(isNonAssigneeWorkspaceBusyRetry("ai_connection_busy", {
|
||||
aiConnectionBusyDeferredWhileAssignee: false,
|
||||
|
||||
@@ -16,6 +16,7 @@ function readNonEmptyString(value: unknown): string | null {
|
||||
}
|
||||
|
||||
export const MAX_TURN_CONTINUATION_RETRY_REASON = "max_turns_continuation";
|
||||
export const COMPUTER_ADMISSION_WAIT_RETRY_REASON = "computer_admission_wait";
|
||||
export const WORKSPACE_BUSY_RETRY_REASON = "workspace_busy";
|
||||
export const AI_CONNECTION_POOL_WAIT_RETRY_REASON = "ai_connection_pool_wait";
|
||||
export const AI_CONNECTION_BUSY_RETRY_REASON = "ai_connection_busy";
|
||||
@@ -40,6 +41,8 @@ export function isNonAssigneeWorkspaceBusyRetry(
|
||||
contextSnapshot: Record<string, unknown>,
|
||||
): boolean {
|
||||
return (
|
||||
(retryReason === COMPUTER_ADMISSION_WAIT_RETRY_REASON &&
|
||||
contextSnapshot.computerAdmissionDeferredWhileAssignee === false) ||
|
||||
(retryReason === WORKSPACE_BUSY_RETRY_REASON &&
|
||||
contextSnapshot.workspaceBusyDeferredWhileAssignee === false) ||
|
||||
((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) &&
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
|
||||
import { executionFailureRetryCount, isComputerAdmissionWait } from "./execution-recovery-attempt.js";
|
||||
|
||||
describe("failure attempts across resource waits", () => {
|
||||
it("preserves prior failures through subscription waits without trusting unrelated context", () => {
|
||||
@@ -27,7 +27,7 @@ describe("failure attempts across resource waits", () => {
|
||||
});
|
||||
|
||||
describe("persisted independent accounting", () => {
|
||||
it.each(["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"])("%s cannot erase prior infrastructure debits or spend more", scheduledRetryReason => {
|
||||
it.each(["computer_admission_wait", "max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"])("%s cannot erase prior infrastructure debits or spend more", scheduledRetryReason => {
|
||||
expect(executionFailureRetryCount({ scheduledRetryReason, scheduledRetryAttempt: 20,
|
||||
contextSnapshot: { executionRetryAccounting: { version: 1, failureRetries: 2, maxTurnContinuations: 1 } },
|
||||
})).toBe(2);
|
||||
@@ -43,3 +43,45 @@ describe("persisted independent accounting", () => {
|
||||
})).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("structural computer admission wait receipt", () => {
|
||||
function receipt() {
|
||||
return {
|
||||
id: "run-1", companyId: "company-1", status: "cancelled", runtimeMode: "legacy",
|
||||
errorCode: "computer_admission_wait",
|
||||
resultJson: {
|
||||
executionRecovery: { kind: "computer_admission_wait", providerWorkStarted: false },
|
||||
cancellation: { expected: true, source: "control_plane", initiator: { type: "system" } },
|
||||
computerAdmission: {
|
||||
runId: "run-1", companyId: "company-1",
|
||||
computerId: "11111111-1111-4111-8111-111111111111",
|
||||
environmentId: "22222222-2222-4222-8222-222222222222", stopId: "stop-1",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("recognizes the recorded never-dispatched wait without treating a code alone as proof", () => {
|
||||
const run = receipt();
|
||||
expect(isComputerAdmissionWait(run)).toBe(true);
|
||||
expect(isComputerAdmissionWait({ ...run, resultJson: {} })).toBe(false);
|
||||
expect(isComputerAdmissionWait({ ...run, resultJson: { ...run.resultJson, executionRecovery: { kind: "computer_admission_wait", providerWorkStarted: true } } })).toBe(false);
|
||||
expect(isComputerAdmissionWait({ ...run, resultJson: { ...run.resultJson, cancellation: { ...run.resultJson.cancellation, initiator: { type: "user" } } } })).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects foreign run/company receipts, invalid identities and already settled retries", () => {
|
||||
const run = receipt();
|
||||
for (const computerAdmission of [
|
||||
{ ...run.resultJson.computerAdmission, runId: "another-run" },
|
||||
{ ...run.resultJson.computerAdmission, companyId: "another-company" },
|
||||
{ ...run.resultJson.computerAdmission, computerId: "invalid" },
|
||||
{ ...run.resultJson.computerAdmission, environmentId: "invalid" },
|
||||
{ ...run.resultJson.computerAdmission, stopId: "" },
|
||||
]) expect(isComputerAdmissionWait({ ...run, resultJson: { ...run.resultJson, computerAdmission } })).toBe(false);
|
||||
expect(isComputerAdmissionWait({ ...run, resultJson: { ...run.resultJson, computerAdmissionRetryOutcome: "superseded" } })).toBe(false);
|
||||
expect(isComputerAdmissionWait({ ...run, runnerProfileJson: { nativeExecutionInput: { version: 1 } } })).toBe(false);
|
||||
expect(isComputerAdmissionWait({ ...run, runtimeMode: "native" })).toBe(false);
|
||||
expect(isComputerAdmissionWait({ ...run, status: "failed" })).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -32,6 +32,10 @@ function historicalFailureCount(run: RetryRun): number {
|
||||
const saved = count(run.contextSnapshot?.failureRetriesBeforeAiConnectionWait);
|
||||
if (saved !== null) return saved;
|
||||
}
|
||||
if (run.scheduledRetryReason === "computer_admission_wait") {
|
||||
const saved = count(run.contextSnapshot?.failureRetriesBeforeComputerWait);
|
||||
if (saved !== null) return saved;
|
||||
}
|
||||
if (run.scheduledRetryReason === "workspace_busy") {
|
||||
const saved = count(run.contextSnapshot?.failureRetriesBeforeWorkspaceWait);
|
||||
if (saved !== null) return saved;
|
||||
@@ -42,7 +46,7 @@ function historicalFailureCount(run: RetryRun): number {
|
||||
|
||||
export function executionRetryAccounting(run: RetryRun): ExecutionRetryAccounting {
|
||||
const saved = savedAccounting(run);
|
||||
const nonFailureLane = ["max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"].includes(run.scheduledRetryReason ?? "");
|
||||
const nonFailureLane = ["computer_admission_wait", "max_turns_continuation", "issue_disposition_repair", "workspace_busy", "ai_connection_busy", "ai_connection_pool_wait"].includes(run.scheduledRetryReason ?? "");
|
||||
return {
|
||||
version: 1,
|
||||
failureRetries: Math.max(saved?.failureRetries ?? 0, saved && nonFailureLane ? 0 : historicalFailureCount(run)),
|
||||
@@ -57,7 +61,7 @@ export function executionFailureRetryCount(run: RetryRun): number {
|
||||
}
|
||||
|
||||
export function executionRetryAttemptCount(run: RetryRun, reason: string): number {
|
||||
if (reason === "workspace_busy" || reason === "ai_connection_busy" || reason === "ai_connection_pool_wait") {
|
||||
if (reason === "computer_admission_wait" || reason === "workspace_busy" || reason === "ai_connection_busy" || reason === "ai_connection_pool_wait") {
|
||||
return run.scheduledRetryReason === reason ? count(run.scheduledRetryAttempt) ?? 0 : 0;
|
||||
}
|
||||
const accounting = executionRetryAccounting(run);
|
||||
@@ -67,6 +71,24 @@ export function executionRetryAttemptCount(run: RetryRun, reason: string): numbe
|
||||
export function accountingForScheduledRetry(run: RetryRun, reason: string, attempt: number): ExecutionRetryAccounting {
|
||||
const accounting = executionRetryAccounting(run);
|
||||
if (reason === "max_turns_continuation") accounting.maxTurnContinuations = attempt;
|
||||
else if (reason !== "workspace_busy" && reason !== "ai_connection_busy" && reason !== "ai_connection_pool_wait") accounting.failureRetries = attempt;
|
||||
else if (reason !== "computer_admission_wait" && reason !== "workspace_busy" && reason !== "ai_connection_busy" && reason !== "ai_connection_pool_wait") accounting.failureRetries = attempt;
|
||||
return accounting;
|
||||
}
|
||||
|
||||
/** Only the executor can attest that a pending-stop wait never dispatched and finished cleanup. */
|
||||
export function isComputerAdmissionWait(run: {
|
||||
id: string; companyId: string; status: string; runtimeMode: string; errorCode?: string | null;
|
||||
resultJson?: Record<string, unknown> | null; runnerProfileJson?: Record<string, unknown> | null;
|
||||
}): boolean {
|
||||
const object = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
const result = object(run.resultJson), recovery = object(result.executionRecovery), cancellation = object(result.cancellation), admission = object(result.computerAdmission);
|
||||
return run.runtimeMode === "legacy" && run.status === "cancelled" && run.errorCode === "computer_admission_wait" &&
|
||||
recovery.kind === "computer_admission_wait" && recovery.providerWorkStarted === false &&
|
||||
result.computerAdmissionRetryOutcome == null &&
|
||||
cancellation.expected === true && cancellation.source === "control_plane" &&
|
||||
object(cancellation.initiator).type === "system" &&
|
||||
admission.runId === run.id && admission.companyId === run.companyId &&
|
||||
[admission.computerId, admission.environmentId].every(value => typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) &&
|
||||
typeof admission.stopId === "string" && admission.stopId.length > 0 &&
|
||||
!object(run.runnerProfileJson).nativeExecutionInput;
|
||||
}
|
||||
Reference in new issue
Block a user