From bc4b3564cddf2bbdcc39f72c1b6550554f0f84f7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 06:18:20 -0500 Subject: [PATCH] fix(e2e): await pending native run recovery Co-Authored-By: Paperclip --- tests/runner-e2e/PRODUCT-JOURNEYS.md | 9 ++ .../onboarding-first-task.test.ts | 95 +++++++++++++++++++ .../product-journeys/onboarding-first-task.ts | 6 +- .../product-journeys/workspace-deferral.ts | 15 +++ 4 files changed, 123 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 2fccdd102bd..c5cf56ff5ce 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -105,6 +105,15 @@ server proves that no provider work started. A scheduled replacement remains pending and consumes the run limit. It cannot satisfy completion; the child must still finish through native execution and pass the independent artifact checks. +A native run with a saved automatic same-run recovery remains pending within the +original campaign deadline. The authenticated run projection must identify an +agent-owned retryable native interruption, a bounded attempt and retry schedule, +and no successor. An overdue retry can still be reconnecting; it does not extend +the deadline or authorize a new story attempt. Ordinary failures, terminal or +board-owned recovery, and exhausted attempts still fail. The producing turn must +settle before proposal acceptance, and the recovered child must actually succeed +before the existing document, artifact, download, and parent-link checks can pass. + The attached `ask-question` recipe requires a prepared native general agent and an explicit execution environment. Legacy execution is unsupported until its saved model/provider identity can be verified. A registered diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts index 02f688b347f..9104bcee9c8 100644 --- a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts @@ -301,6 +301,93 @@ describe("Cloud canonical onboarding first task", () => { expect(result.artifact.originatingRunId).toBe("child-run"); expect(connectionDelay).toHaveBeenCalledOnce(); }); + it.each([false, true])("observes the same native recovery through success with workspace predecessor=%s", async predecessor => { + let now = Date.now(), stage = 0; + const failedAt = now - 1000, retryAt = now + 30_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + vi.mocked(connectionDelay).mockImplementation(async () => { stage++; now = stage === 1 ? retryAt + 1000 : now + 1000; }); + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + const child = state.runs.find(run => run.id === "child-run")!; + if (predecessor) { state.runs.splice(state.runs.indexOf(child), 0, workspaceDeferral()); linkWorkspaceRetry(child); } + if (stage < 2) { + withPendingNativeRecovery(child, failedAt, retryAt); + child.execution.phase = stage === 0 ? "retry_scheduled" : "reconnecting"; + child.scheduledRetryAt = new Date(failedAt - 60_000).toISOString(); + } else if (stage === 2) Object.assign(child, { status: "running", nativePhase: "observed", finishedAt: null }); + } }); + const result = await onboardingFirstTask(f.input); + expect(stage).toBe(3); + expect(result.artifact.originatingRunId).toBe("child-run"); + expect(result.runIds).toHaveLength(predecessor ? 5 : 4); + expect(f.events.filter(event => event === "click:Continue")).toHaveLength(1); + expect(f.events.filter(event => event === "click:Approve")).toHaveLength(1); + expect(f.events).toContain("visible:download"); + }); + it("waits for a recovering proposal turn before the single human acceptance", async () => { + let now = Date.now(), stage = 0; + const failedAt = now - 1000, retryAt = now + 30_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + let f: ReturnType; + vi.mocked(connectionDelay).mockImplementation(async () => { + expect(f.events).not.toContain("click:Approve"); + stage++; now = retryAt + stage * 1000; + }); + f = fixture({ mutate(phase, state) { + if (phase !== "proposal" || stage >= 2) return; + withPendingNativeRecovery(state.runs[0]!, failedAt, retryAt); + state.runs[0]!.execution.phase = stage === 0 ? "retry_scheduled" : "reconnecting"; + } }); + await expect(onboardingFirstTask(f.input)).resolves.toMatchObject({ artifact: { originatingRunId: "child-run" } }); + expect(stage).toBe(2); + expect(f.events.filter(event => event === "click:Approve")).toHaveLength(1); + }); + it("never completes from a pending native recovery after the original campaign deadline", async () => { + let now = Date.now(); + const failedAt = now - 1000, retryAt = now + 1000; + vi.spyOn(Date, "now").mockImplementation(() => now); + vi.mocked(connectionDelay).mockImplementation(async ms => { now += ms; }); + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + const child = state.runs.find(run => run.id === "child-run")!; + withPendingNativeRecovery(child, failedAt, retryAt); + child.execution.phase = now < retryAt ? "retry_scheduled" : "reconnecting"; + } }); + f.input.deadlineAt = now + 3000; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_deadline" }); + expect(f.events).not.toContain("visible:download"); + }); + it.each([ + ["missing projection", (run: Row) => { delete run.execution; }], + ["legacy execution", (run: Row) => { run.runtimeMode = "legacy"; }], + ["terminal failure", (run: Row) => { run.nativePhase = "terminal_failure"; }], + ["missing native issue", (run: Row) => { run.nativeIssueId = null; }], + ["mismatched issue binding", (run: Row) => { run.contextSnapshot.issueId = "other-issue"; }], + ["provider failure", (run: Row) => { run.errorCode = run.execution.cause = "native_provider_terminal_failed"; }], + ["mismatched cause", (run: Row) => { run.execution.cause = "provider_transport_failed"; }], + ["generic recovery", (run: Row) => { run.execution.phase = "recovery_needed"; }], + ["board owner", (run: Row) => { run.execution.recoveryOwner = "board"; }], + ["missing schedule", (run: Row) => { run.execution.retryAt = null; }], + ["malformed schedule", (run: Row) => { run.execution.retryAt = "invalid"; }], + ["old workspace schedule", (run: Row) => { run.execution.retryAt = new Date(Date.parse(run.finishedAt) - 1).toISOString(); }], + ["out-of-campaign schedule", (run: Row) => { run.execution.retryAt = new Date(Date.now() + 60_001).toISOString(); }], + ["exhausted attempts", (run: Row) => { run.execution.attempt = 3; }], + ["missing attempt", (run: Row) => { delete run.execution.attempt; }], + ["noninteger attempt", (run: Row) => { run.execution.attempt = 1.5; }], + ["unexpected ceiling", (run: Row) => { run.execution.maxAttempts = 4; }], + ["different successor", (run: Row) => { run.execution.successorRunId = "other-run"; }], + ["missing successor state", (run: Row) => { delete run.execution.successorRunId; }], + ] as const)("keeps a failed native run fatal with %s", async (_label, change) => { + const now = Date.now(); + vi.spyOn(Date, "now").mockReturnValue(now); + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + const child = state.runs.find(run => run.id === "child-run")!; + withPendingNativeRecovery(child, now - 1000, now + 30_000); change(child); + } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" }); + expect(f.events).not.toContain("visible:download"); + }); it.each(["transient_failure", "ai_connection_busy", "ai_connection_pool_wait", "max_turns_continuation", "issue_disposition_repair"])("starts the workspace wait counter independently of %s", async reason => { const f = fixture({ mutate(phase, state) { if (phase !== "accepted") return; @@ -438,3 +525,11 @@ function workspaceDeferral(): Row { cancellation: { source: "control_plane", expected: true, initiator: { type: "system" } }, } }; } + +function withPendingNativeRecovery(run: Row, failedAt: number, retryAt: number) { + Object.assign(run, { status: "failed", nativePhase: "retryable_failure", errorCode: "native_session_interrupted", + finishedAt: new Date(failedAt).toISOString(), execution: { + phase: "retry_scheduled", cause: "native_session_interrupted", retryAt: new Date(retryAt).toISOString(), + attempt: 1, maxAttempts: 3, recoveryOwner: "agent", successorRunId: null, + } }); +} diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.ts index e1ecf63fb1e..4c5d72e946f 100644 --- a/tests/runner-e2e/product-journeys/onboarding-first-task.ts +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.ts @@ -7,7 +7,7 @@ import { isBlockedUnstartedWake, isTerminalUnstartedWake } from "../non-executio import { JourneyError, type JourneyContext, type OnboardingArtifactEvidence, type TargetSession } from "./types.js"; import { retainOnboardingArtifact } from "./artifact-evidence.js"; import { preAcceptanceTaskStatus, preAcceptanceViolations, type OnboardingPreAcceptanceEvidence } from "./onboarding-pre-acceptance.js"; -import { isProviderUnstartedWorkspaceDeferral } from "./workspace-deferral.js"; +import { isPendingNativeRecovery, isProviderUnstartedWorkspaceDeferral } from "./workspace-deferral.js"; export interface OnboardingFirstTaskInput { context: JourneyContext; @@ -41,7 +41,6 @@ function fail(code: string): never { throw new JourneyError("failed", code); } const runIssueId = (run: Row) => run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId; const isProposal = (row: Row) => ["request_confirmation", "request_checkbox_confirmation"].includes(row.kind); const isPlanningDocument = (row: Row) => /^(?:plan|proposal)$/.test(row.key); -const isPendingRun = (run: Row) => activeRuns([run]).length > 0 || run.status === "scheduled_retry"; function sameWorkspaceRetry(source: Row, successor: Row): boolean { const issueIds = (run: Row) => [run.nativeIssueId, run.issueId, run.contextSnapshot?.issueId].filter(value => value != null); @@ -65,6 +64,8 @@ function unstartedScheduledWorkspaceRetry(run: Row): boolean { /** Exercise the ordinary onboarding conversation; APIs observe, never submit its actions. */ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Promise { const { context: { api, page, company }, agent, campaignId, proof } = input; + const isPendingRun = (run: Row) => activeRuns([run]).length > 0 || run.status === "scheduled_retry" || + isPendingNativeRecovery(run, input.deadlineAt); if (!Number.isSafeInteger(input.maxRuns) || input.maxRuns < 1) fail("invalid_onboarding_run_limit"); const title = `Artifact ${campaignId}`; const seen = new Map(); @@ -136,6 +137,7 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom workspacePending = true; continue; } + if (isPendingNativeRecovery(run, input.deadlineAt)) { workspacePending = true; continue; } if (activeRuns([run]).length || run.status === "succeeded" || isBlockedUnstartedWake(run) || isTerminalUnstartedWake(run, snapshot.tasks)) continue; fail("onboarding_first_task_failed"); } diff --git a/tests/runner-e2e/product-journeys/workspace-deferral.ts b/tests/runner-e2e/product-journeys/workspace-deferral.ts index 2a831c25545..ba0a66b8461 100644 --- a/tests/runner-e2e/product-journeys/workspace-deferral.ts +++ b/tests/runner-e2e/product-journeys/workspace-deferral.ts @@ -15,6 +15,21 @@ function hasProviderUsage(value: unknown): boolean { }); } +export function isPendingNativeRecovery(run: RecordValue, deadlineAt: number): boolean { + const execution = object(run.execution); + const finishedAt = typeof run.finishedAt === "string" ? Date.parse(run.finishedAt) : NaN; + const retryAt = typeof execution.retryAt === "string" ? Date.parse(execution.retryAt) : NaN; + return run.runtimeMode === "native" && run.status === "failed" && run.nativePhase === "retryable_failure" && + run.errorCode === "native_session_interrupted" && execution.cause === "native_session_interrupted" && + typeof run.nativeIssueId === "string" && run.nativeIssueId === object(run.contextSnapshot).issueId && + (run.issueId == null || run.issueId === run.nativeIssueId) && + (execution.phase === "retry_scheduled" || execution.phase === "reconnecting") && execution.recoveryOwner === "agent" && + Number.isInteger(execution.attempt) && Number(execution.attempt) >= 1 && + execution.maxAttempts === 3 && Number(execution.attempt) < execution.maxAttempts && execution.successorRunId === null && + Number.isFinite(finishedAt) && Number.isFinite(retryAt) && Number.isFinite(deadlineAt) && + retryAt > finishedAt && retryAt <= deadlineAt; +} + /** Admission may set startedAt before selecting a runtime. Only this explicit * control-plane receipt, without contradictory provider evidence, proves no turn * ran. A legacy default alone is never proof of either execution or nonexecution. */