fix(e2e): follow server-owned recovery state

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 06:39:46 -05:00
1 parent b17ecd1946
commit 04dedd7d6e
3 files changed
+45 -12

No files matched your search

+7 -7
View File
@@ -105,13 +105,13 @@ server proves that no provider work started. A scheduled replacement remains
pending and consumes the run limit. It cannot satisfy completion; the child must pending and consumes the run limit. It cannot satisfy completion; the child must
still finish through native execution and pass the independent artifact checks. still finish through native execution and pass the independent artifact checks.
A native run can remain pending for saved automatic same-run recovery with code A failed native run can remain pending when the authenticated run projection
`native_session_interrupted`, `provider_transport_failed`, identifies agent-owned automatic recovery on that same run. Its native phase
`provider_request_timeout`, or `native_runner_process_exited`. must remain retryable, with matching nonempty cause strings, a bounded attempt
The wait stays within the original campaign deadline. and retry schedule, and no successor. Cause strings are diagnostic values, not
The authenticated run projection must identify a matching source code and an a positive retry policy. Known permanent, permission-declined, and exhausted
agent-owned retryable native interruption, a bounded attempt and retry schedule, causes are also rejected defensively. The wait stays within the original
and no successor. An overdue retry can still be reconnecting; it does not extend campaign deadline. An overdue retry can still be reconnecting; it does not extend
the deadline or authorize a new story attempt. Ordinary failures, terminal or the deadline or authorize a new story attempt. Ordinary failures, terminal or
board-owned recovery, and exhausted attempts still fail. The producing turn must board-owned recovery, and exhausted attempts still fail. The producing turn must
settle before proposal acceptance, and the recovered child must actually succeed settle before proposal acceptance, and the recovered child must actually succeed
@@ -301,7 +301,10 @@ describe("Cloud canonical onboarding first task", () => {
expect(result.artifact.originatingRunId).toBe("child-run"); expect(result.artifact.originatingRunId).toBe("child-run");
expect(connectionDelay).toHaveBeenCalledOnce(); expect(connectionDelay).toHaveBeenCalledOnce();
}); });
it.each(["native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited"] it.each(["native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited",
"provider_process_exited", "provider_stdout_closed", "provider_process_output_closed", "provider_process_status_failed",
"provider_initialize_timeout", "provider_initialize_protocol_error", "provider_request_protocol_error",
"provider_frame_too_large", "planning_mode_unsupported", "future_custom_recovery"]
.flatMap(cause => [false, true].map(predecessor => ({ cause, predecessor }))))("observes $cause recovery through success with workspace predecessor=$predecessor", async ({ cause, predecessor }) => { .flatMap(cause => [false, true].map(predecessor => ({ cause, predecessor }))))("observes $cause recovery through success with workspace predecessor=$predecessor", async ({ cause, predecessor }) => {
let now = Date.now(), stage = 0; let now = Date.now(), stage = 0;
const failedAt = now - 1000, retryAt = now + 30_000; const failedAt = now - 1000, retryAt = now + 30_000;
@@ -366,7 +369,15 @@ describe("Cloud canonical onboarding first task", () => {
["mismatched issue binding", (run: Row) => { run.contextSnapshot.issueId = "other-issue"; }], ["mismatched issue binding", (run: Row) => { run.contextSnapshot.issueId = "other-issue"; }],
["provider failure", (run: Row) => { run.errorCode = run.execution.cause = "native_provider_terminal_failed"; }], ["provider failure", (run: Row) => { run.errorCode = run.execution.cause = "native_provider_terminal_failed"; }],
["mismatched cause", (run: Row) => { run.execution.cause = "provider_transport_failed"; }], ["mismatched cause", (run: Row) => { run.execution.cause = "provider_transport_failed"; }],
["unknown matching cause", (run: Row) => { run.errorCode = run.execution.cause = "unclassified_provider_error"; }], ["empty matching cause", (run: Row) => { run.errorCode = run.execution.cause = ""; }],
["blank matching cause", (run: Row) => { run.errorCode = run.execution.cause = " "; }],
["missing run cause", (run: Row) => { delete run.errorCode; }],
["missing execution cause", (run: Row) => { delete run.execution.cause; }],
["nonstrings matching cause", (run: Row) => { run.errorCode = run.execution.cause = 42; }],
["terminal provider checking recovery", (run: Row) => {
run.errorCode = run.execution.cause = "native_provider_terminal_failed"; run.nativePhase = "terminal_failure";
Object.assign(run.execution, { phase: "reconnecting", recoveryOwner: "agent", retryAt: null });
}],
["ambiguous transport failure", (run: Row) => { ["ambiguous transport failure", (run: Row) => {
run.errorCode = run.execution.cause = "provider_transport_failed"; run.nativePhase = "terminal_failure"; run.errorCode = run.execution.cause = "provider_transport_failed"; run.nativePhase = "terminal_failure";
Object.assign(run.execution, { phase: "recovery_needed", recoveryOwner: "board", retryAt: null }); Object.assign(run.execution, { phase: "recovery_needed", recoveryOwner: "board", retryAt: null });
@@ -394,6 +405,22 @@ describe("Cloud canonical onboarding first task", () => {
await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" }); await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" });
expect(f.events).not.toContain("visible:download"); expect(f.events).not.toContain("visible:download");
}); });
it.each([
"native_provider_runtime_missing", "native_provider_model_rejected", "native_provider_approval_required",
"native_event_replay_conflict", "runner_remote_provider_artifact_incompatible",
"runner_remote_provider_verification_unavailable", "native_current_wake_comments_unread",
"native_current_wake_comments_changed_after_read", "native_session_cleanup_quarantined",
"native_adopted_runner_authentication_timeout", "native_provider_usage_limit",
"native_permission_declined", "native_session_retry_exhausted",
])("rejects permanent %s even when other fields claim a pending retry", async cause => {
const now = Date.now();
vi.spyOn(Date, "now").mockReturnValue(now);
const f = fixture({ mutate(phase, state) {
if (phase === "accepted") withPendingNativeRecovery(state.runs.find(run => run.id === "child-run")!, now - 1000, now + 30_000, cause);
} });
await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" });
expect(f.events).not.toContain("visible:download");
});
it.each(["transient_failure", "ai_connection_busy", "ai_connection_pool_wait", "max_turns_continuation", "issue_disposition_repair"])("starts the workspace wait counter independently of %s", async reason => { it.each(["transient_failure", "ai_connection_busy", "ai_connection_pool_wait", "max_turns_continuation", "issue_disposition_repair"])("starts the workspace wait counter independently of %s", async reason => {
const f = fixture({ mutate(phase, state) { const f = fixture({ mutate(phase, state) {
if (phase !== "accepted") return; if (phase !== "accepted") return;
@@ -1,8 +1,13 @@
type RecordValue = Record<string, unknown>; type RecordValue = Record<string, unknown>;
const object = (value: unknown): RecordValue => value !== null && typeof value === "object" && !Array.isArray(value) const object = (value: unknown): RecordValue => value !== null && typeof value === "object" && !Array.isArray(value)
? value as RecordValue : {}; ? value as RecordValue : {};
const nativeRecoveryCauses = new Set([ const nativeRecoveryTerminalCauses = new Set([
"native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited", "native_provider_runtime_missing", "native_provider_model_rejected", "native_provider_approval_required",
"native_event_replay_conflict", "runner_remote_provider_artifact_incompatible",
"runner_remote_provider_verification_unavailable", "native_provider_terminal_failed",
"native_current_wake_comments_unread", "native_current_wake_comments_changed_after_read",
"native_session_cleanup_quarantined", "native_adopted_runner_authentication_timeout", "native_provider_usage_limit",
"native_permission_declined", "native_session_retry_exhausted",
]); ]);
function hasProviderUsage(value: unknown): boolean { function hasProviderUsage(value: unknown): boolean {
@@ -23,7 +28,8 @@ export function isPendingNativeRecovery(run: RecordValue, deadlineAt: number): b
const finishedAt = typeof run.finishedAt === "string" ? Date.parse(run.finishedAt) : NaN; const finishedAt = typeof run.finishedAt === "string" ? Date.parse(run.finishedAt) : NaN;
const retryAt = typeof execution.retryAt === "string" ? Date.parse(execution.retryAt) : NaN; const retryAt = typeof execution.retryAt === "string" ? Date.parse(execution.retryAt) : NaN;
return run.runtimeMode === "native" && run.status === "failed" && run.nativePhase === "retryable_failure" && return run.runtimeMode === "native" && run.status === "failed" && run.nativePhase === "retryable_failure" &&
typeof run.errorCode === "string" && nativeRecoveryCauses.has(run.errorCode) && execution.cause === run.errorCode && typeof run.errorCode === "string" && run.errorCode.trim().length > 0 && execution.cause === run.errorCode &&
!nativeRecoveryTerminalCauses.has(run.errorCode) &&
typeof run.nativeIssueId === "string" && run.nativeIssueId === object(run.contextSnapshot).issueId && typeof run.nativeIssueId === "string" && run.nativeIssueId === object(run.contextSnapshot).issueId &&
(run.issueId == null || run.issueId === run.nativeIssueId) && (run.issueId == null || run.issueId === run.nativeIssueId) &&
(execution.phase === "retry_scheduled" || execution.phase === "reconnecting") && execution.recoveryOwner === "agent" && (execution.phase === "retry_scheduled" || execution.phase === "reconnecting") && execution.recoveryOwner === "agent" &&