fix(e2e): follow server-owned recovery state

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 06:39:46 -05:00
1 parent b17ecd1946
commit 04dedd7d6e
3 files changed
+45 -12

No files matched your search

+7 -7
View File
@@ -105,13 +105,13 @@ server proves that no provider work started. A scheduled replacement remains
pending and consumes the run limit. It cannot satisfy completion; the child must
still finish through native execution and pass the independent artifact checks.
A native run can remain pending for saved automatic same-run recovery with code
`native_session_interrupted`, `provider_transport_failed`,
`provider_request_timeout`, or `native_runner_process_exited`.
The wait stays within the original campaign deadline.
The authenticated run projection must identify a matching source code and an
agent-owned retryable native interruption, a bounded attempt and retry schedule,
and no successor. An overdue retry can still be reconnecting; it does not extend
A failed native run can remain pending when the authenticated run projection
identifies agent-owned automatic recovery on that same run. Its native phase
must remain retryable, with matching nonempty cause strings, a bounded attempt
and retry schedule, and no successor. Cause strings are diagnostic values, not
a positive retry policy. Known permanent, permission-declined, and exhausted
causes are also rejected defensively. The wait stays within the original
campaign deadline. An overdue retry can still be reconnecting; it does not extend
the deadline or authorize a new story attempt. Ordinary failures, terminal or
board-owned recovery, and exhausted attempts still fail. The producing turn must
settle before proposal acceptance, and the recovered child must actually succeed
@@ -301,7 +301,10 @@ describe("Cloud canonical onboarding first task", () => {
expect(result.artifact.originatingRunId).toBe("child-run");
expect(connectionDelay).toHaveBeenCalledOnce();
});
it.each(["native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited"]
it.each(["native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited",
"provider_process_exited", "provider_stdout_closed", "provider_process_output_closed", "provider_process_status_failed",
"provider_initialize_timeout", "provider_initialize_protocol_error", "provider_request_protocol_error",
"provider_frame_too_large", "planning_mode_unsupported", "future_custom_recovery"]
.flatMap(cause => [false, true].map(predecessor => ({ cause, predecessor }))))("observes $cause recovery through success with workspace predecessor=$predecessor", async ({ cause, predecessor }) => {
let now = Date.now(), stage = 0;
const failedAt = now - 1000, retryAt = now + 30_000;
@@ -366,7 +369,15 @@ describe("Cloud canonical onboarding first task", () => {
["mismatched issue binding", (run: Row) => { run.contextSnapshot.issueId = "other-issue"; }],
["provider failure", (run: Row) => { run.errorCode = run.execution.cause = "native_provider_terminal_failed"; }],
["mismatched cause", (run: Row) => { run.execution.cause = "provider_transport_failed"; }],
["unknown matching cause", (run: Row) => { run.errorCode = run.execution.cause = "unclassified_provider_error"; }],
["empty matching cause", (run: Row) => { run.errorCode = run.execution.cause = ""; }],
["blank matching cause", (run: Row) => { run.errorCode = run.execution.cause = " "; }],
["missing run cause", (run: Row) => { delete run.errorCode; }],
["missing execution cause", (run: Row) => { delete run.execution.cause; }],
["nonstrings matching cause", (run: Row) => { run.errorCode = run.execution.cause = 42; }],
["terminal provider checking recovery", (run: Row) => {
run.errorCode = run.execution.cause = "native_provider_terminal_failed"; run.nativePhase = "terminal_failure";
Object.assign(run.execution, { phase: "reconnecting", recoveryOwner: "agent", retryAt: null });
}],
["ambiguous transport failure", (run: Row) => {
run.errorCode = run.execution.cause = "provider_transport_failed"; run.nativePhase = "terminal_failure";
Object.assign(run.execution, { phase: "recovery_needed", recoveryOwner: "board", retryAt: null });
@@ -394,6 +405,22 @@ describe("Cloud canonical onboarding first task", () => {
await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" });
expect(f.events).not.toContain("visible:download");
});
it.each([
"native_provider_runtime_missing", "native_provider_model_rejected", "native_provider_approval_required",
"native_event_replay_conflict", "runner_remote_provider_artifact_incompatible",
"runner_remote_provider_verification_unavailable", "native_current_wake_comments_unread",
"native_current_wake_comments_changed_after_read", "native_session_cleanup_quarantined",
"native_adopted_runner_authentication_timeout", "native_provider_usage_limit",
"native_permission_declined", "native_session_retry_exhausted",
])("rejects permanent %s even when other fields claim a pending retry", async cause => {
const now = Date.now();
vi.spyOn(Date, "now").mockReturnValue(now);
const f = fixture({ mutate(phase, state) {
if (phase === "accepted") withPendingNativeRecovery(state.runs.find(run => run.id === "child-run")!, now - 1000, now + 30_000, cause);
} });
await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" });
expect(f.events).not.toContain("visible:download");
});
it.each(["transient_failure", "ai_connection_busy", "ai_connection_pool_wait", "max_turns_continuation", "issue_disposition_repair"])("starts the workspace wait counter independently of %s", async reason => {
const f = fixture({ mutate(phase, state) {
if (phase !== "accepted") return;
@@ -1,8 +1,13 @@
type RecordValue = Record<string, unknown>;
const object = (value: unknown): RecordValue => value !== null && typeof value === "object" && !Array.isArray(value)
? value as RecordValue : {};
const nativeRecoveryCauses = new Set([
"native_session_interrupted", "provider_transport_failed", "provider_request_timeout", "native_runner_process_exited",
const nativeRecoveryTerminalCauses = new Set([
"native_provider_runtime_missing", "native_provider_model_rejected", "native_provider_approval_required",
"native_event_replay_conflict", "runner_remote_provider_artifact_incompatible",
"runner_remote_provider_verification_unavailable", "native_provider_terminal_failed",
"native_current_wake_comments_unread", "native_current_wake_comments_changed_after_read",
"native_session_cleanup_quarantined", "native_adopted_runner_authentication_timeout", "native_provider_usage_limit",
"native_permission_declined", "native_session_retry_exhausted",
]);
function hasProviderUsage(value: unknown): boolean {
@@ -23,7 +28,8 @@ export function isPendingNativeRecovery(run: RecordValue, deadlineAt: number): b
const finishedAt = typeof run.finishedAt === "string" ? Date.parse(run.finishedAt) : NaN;
const retryAt = typeof execution.retryAt === "string" ? Date.parse(execution.retryAt) : NaN;
return run.runtimeMode === "native" && run.status === "failed" && run.nativePhase === "retryable_failure" &&
typeof run.errorCode === "string" && nativeRecoveryCauses.has(run.errorCode) && execution.cause === run.errorCode &&
typeof run.errorCode === "string" && run.errorCode.trim().length > 0 && execution.cause === run.errorCode &&
!nativeRecoveryTerminalCauses.has(run.errorCode) &&
typeof run.nativeIssueId === "string" && run.nativeIssueId === object(run.contextSnapshot).issueId &&
(run.issueId == null || run.issueId === run.nativeIssueId) &&
(execution.phase === "retry_scheduled" || execution.phase === "reconnecting") && execution.recoveryOwner === "agent" &&