From 04dedd7d6e23408f1cee9334f1007d85e23bbda0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 06:39:46 -0500 Subject: [PATCH] fix(e2e): follow server-owned recovery state Co-Authored-By: Paperclip --- tests/runner-e2e/PRODUCT-JOURNEYS.md | 14 ++++----- .../onboarding-first-task.test.ts | 31 +++++++++++++++++-- .../product-journeys/workspace-deferral.ts | 12 +++++-- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 3d1f998169a..296dbc6309f 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -105,13 +105,13 @@ server proves that no provider work started. A scheduled replacement remains pending and consumes the run limit. It cannot satisfy completion; the child must still finish through native execution and pass the independent artifact checks. -A native run can remain pending for saved automatic same-run recovery with code -`native_session_interrupted`, `provider_transport_failed`, -`provider_request_timeout`, or `native_runner_process_exited`. -The wait stays within the original campaign deadline. -The authenticated run projection must identify a matching source code and an -agent-owned retryable native interruption, a bounded attempt and retry schedule, -and no successor. An overdue retry can still be reconnecting; it does not extend +A failed native run can remain pending when the authenticated run projection +identifies agent-owned automatic recovery on that same run. Its native phase +must remain retryable, with matching nonempty cause strings, a bounded attempt +and retry schedule, and no successor. Cause strings are diagnostic values, not +a positive retry policy. Known permanent, permission-declined, and exhausted +causes are also rejected defensively. The wait stays within the original +campaign deadline. An overdue retry can still be reconnecting; it does not extend the deadline or authorize a new story attempt. Ordinary failures, terminal or board-owned recovery, and exhausted attempts still fail. The producing turn must settle before proposal acceptance, and the recovered child must actually succeed diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts index 0efa6442d29..8d54258e250 100644 --- a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts @@ -301,7 +301,10 @@ describe("Cloud canonical onboarding first task", () => { expect(result.artifact.originatingRunId).toBe("child-run"); expect(connectionDelay).toHaveBeenCalledOnce(); }); - it.each(["native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited"] + it.each(["native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited", + "provider_process_exited", "provider_stdout_closed", "provider_process_output_closed", "provider_process_status_failed", + "provider_initialize_timeout", "provider_initialize_protocol_error", "provider_request_protocol_error", + "provider_frame_too_large", "planning_mode_unsupported", "future_custom_recovery"] .flatMap(cause => [false, true].map(predecessor => ({ cause, predecessor }))))("observes $cause recovery through success with workspace predecessor=$predecessor", async ({ cause, predecessor }) => { let now = Date.now(), stage = 0; const failedAt = now - 1000, retryAt = now + 30_000; @@ -366,7 +369,15 @@ describe("Cloud canonical onboarding first task", () => { ["mismatched issue binding", (run: Row) => { run.contextSnapshot.issueId = "other-issue"; }], ["provider failure", (run: Row) => { run.errorCode = run.execution.cause = "native_provider_terminal_failed"; }], ["mismatched cause", (run: Row) => { run.execution.cause = "provider_transport_failed"; }], - ["unknown matching cause", (run: Row) => { run.errorCode = run.execution.cause = "unclassified_provider_error"; }], + ["empty matching cause", (run: Row) => { run.errorCode = run.execution.cause = ""; }], + ["blank matching cause", (run: Row) => { run.errorCode = run.execution.cause = " "; }], + ["missing run cause", (run: Row) => { delete run.errorCode; }], + ["missing execution cause", (run: Row) => { delete run.execution.cause; }], + ["nonstrings matching cause", (run: Row) => { run.errorCode = run.execution.cause = 42; }], + ["terminal provider checking recovery", (run: Row) => { + run.errorCode = run.execution.cause = "native_provider_terminal_failed"; run.nativePhase = "terminal_failure"; + Object.assign(run.execution, { phase: "reconnecting", recoveryOwner: "agent", retryAt: null }); + }], ["ambiguous transport failure", (run: Row) => { run.errorCode = run.execution.cause = "provider_transport_failed"; run.nativePhase = "terminal_failure"; Object.assign(run.execution, { phase: "recovery_needed", recoveryOwner: "board", retryAt: null }); @@ -394,6 +405,22 @@ describe("Cloud canonical onboarding first task", () => { await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" }); expect(f.events).not.toContain("visible:download"); }); + it.each([ + "native_provider_runtime_missing", "native_provider_model_rejected", "native_provider_approval_required", + "native_event_replay_conflict", "runner_remote_provider_artifact_incompatible", + "runner_remote_provider_verification_unavailable", "native_current_wake_comments_unread", + "native_current_wake_comments_changed_after_read", "native_session_cleanup_quarantined", + "native_adopted_runner_authentication_timeout", "native_provider_usage_limit", + "native_permission_declined", "native_session_retry_exhausted", + ])("rejects permanent %s even when other fields claim a pending retry", async cause => { + const now = Date.now(); + vi.spyOn(Date, "now").mockReturnValue(now); + const f = fixture({ mutate(phase, state) { + if (phase === "accepted") withPendingNativeRecovery(state.runs.find(run => run.id === "child-run")!, now - 1000, now + 30_000, cause); + } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" }); + expect(f.events).not.toContain("visible:download"); + }); it.each(["transient_failure", "ai_connection_busy", "ai_connection_pool_wait", "max_turns_continuation", "issue_disposition_repair"])("starts the workspace wait counter independently of %s", async reason => { const f = fixture({ mutate(phase, state) { if (phase !== "accepted") return; diff --git a/tests/runner-e2e/product-journeys/workspace-deferral.ts b/tests/runner-e2e/product-journeys/workspace-deferral.ts index 9f004f7a7ad..9146cb87e17 100644 --- a/tests/runner-e2e/product-journeys/workspace-deferral.ts +++ b/tests/runner-e2e/product-journeys/workspace-deferral.ts @@ -1,8 +1,13 @@ type RecordValue = Record; const object = (value: unknown): RecordValue => value !== null && typeof value === "object" && !Array.isArray(value) ? value as RecordValue : {}; -const nativeRecoveryCauses = new Set([ - "native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited", +const nativeRecoveryTerminalCauses = new Set([ + "native_provider_runtime_missing", "native_provider_model_rejected", "native_provider_approval_required", + "native_event_replay_conflict", "runner_remote_provider_artifact_incompatible", + "runner_remote_provider_verification_unavailable", "native_provider_terminal_failed", + "native_current_wake_comments_unread", "native_current_wake_comments_changed_after_read", + "native_session_cleanup_quarantined", "native_adopted_runner_authentication_timeout", "native_provider_usage_limit", + "native_permission_declined", "native_session_retry_exhausted", ]); function hasProviderUsage(value: unknown): boolean { @@ -23,7 +28,8 @@ export function isPendingNativeRecovery(run: RecordValue, deadlineAt: number): b const finishedAt = typeof run.finishedAt === "string" ? Date.parse(run.finishedAt) : NaN; const retryAt = typeof execution.retryAt === "string" ? Date.parse(execution.retryAt) : NaN; return run.runtimeMode === "native" && run.status === "failed" && run.nativePhase === "retryable_failure" && - typeof run.errorCode === "string" && nativeRecoveryCauses.has(run.errorCode) && execution.cause === run.errorCode && + typeof run.errorCode === "string" && run.errorCode.trim().length > 0 && execution.cause === run.errorCode && + !nativeRecoveryTerminalCauses.has(run.errorCode) && typeof run.nativeIssueId === "string" && run.nativeIssueId === object(run.contextSnapshot).issueId && (run.issueId == null || run.issueId === run.nativeIssueId) && (execution.phase === "retry_scheduled" || execution.phase === "reconnecting") && execution.recoveryOwner === "agent" &&