fix(e2e): await pending native run recovery

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/PRODUCT-JOURNEYS.md

@@ -105,6 +105,15 @@ server proves that no provider work started. A scheduled replacement remains
 pending and consumes the run limit. It cannot satisfy completion; the child must
 still finish through native execution and pass the independent artifact checks.
 
+A native run with a saved automatic same-run recovery remains pending within the
+original campaign deadline. The authenticated run projection must identify an
+agent-owned retryable native interruption, a bounded attempt and retry schedule,
+and no successor. An overdue retry can still be reconnecting; it does not extend
+the deadline or authorize a new story attempt. Ordinary failures, terminal or
+board-owned recovery, and exhausted attempts still fail. The producing turn must
+settle before proposal acceptance, and the recovered child must actually succeed
+before the existing document, artifact, download, and parent-link checks can pass.
+
 The attached `ask-question` recipe requires a prepared native general agent and
 an explicit execution environment. Legacy execution is unsupported until its
 saved model/provider identity can be verified. A registered

tests/runner-e2e/product-journeys/onboarding-first-task.test.ts

@@ -301,6 +301,93 @@ describe("Cloud canonical onboarding first task", () => {
     expect(result.artifact.originatingRunId).toBe("child-run");
     expect(connectionDelay).toHaveBeenCalledOnce();
   });
+  it.each([false, true])("observes the same native recovery through success with workspace predecessor=%s", async predecessor => {
+    let now = Date.now(), stage = 0;
+    const failedAt = now - 1000, retryAt = now + 30_000;
+    vi.spyOn(Date, "now").mockImplementation(() => now);
+    vi.mocked(connectionDelay).mockImplementation(async () => { stage++; now = stage === 1 ? retryAt + 1000 : now + 1000; });
+    const f = fixture({ mutate(phase, state) {
+      if (phase !== "accepted") return;
+      const child = state.runs.find(run => run.id === "child-run")!;
+      if (predecessor) { state.runs.splice(state.runs.indexOf(child), 0, workspaceDeferral()); linkWorkspaceRetry(child); }
+      if (stage < 2) {
+        withPendingNativeRecovery(child, failedAt, retryAt);
+        child.execution.phase = stage === 0 ? "retry_scheduled" : "reconnecting";
+        child.scheduledRetryAt = new Date(failedAt - 60_000).toISOString();
+      } else if (stage === 2) Object.assign(child, { status: "running", nativePhase: "observed", finishedAt: null });
+    } });
+    const result = await onboardingFirstTask(f.input);
+    expect(stage).toBe(3);
+    expect(result.artifact.originatingRunId).toBe("child-run");
+    expect(result.runIds).toHaveLength(predecessor ? 5 : 4);
+    expect(f.events.filter(event => event === "click:Continue")).toHaveLength(1);
+    expect(f.events.filter(event => event === "click:Approve")).toHaveLength(1);
+    expect(f.events).toContain("visible:download");
+  });
+  it("waits for a recovering proposal turn before the single human acceptance", async () => {
+    let now = Date.now(), stage = 0;
+    const failedAt = now - 1000, retryAt = now + 30_000;
+    vi.spyOn(Date, "now").mockImplementation(() => now);
+    let f: ReturnType<typeof fixture>;
+    vi.mocked(connectionDelay).mockImplementation(async () => {
+      expect(f.events).not.toContain("click:Approve");
+      stage++; now = retryAt + stage * 1000;
+    });
+    f = fixture({ mutate(phase, state) {
+      if (phase !== "proposal" || stage >= 2) return;
+      withPendingNativeRecovery(state.runs[0]!, failedAt, retryAt);
+      state.runs[0]!.execution.phase = stage === 0 ? "retry_scheduled" : "reconnecting";
+    } });
+    await expect(onboardingFirstTask(f.input)).resolves.toMatchObject({ artifact: { originatingRunId: "child-run" } });
+    expect(stage).toBe(2);
+    expect(f.events.filter(event => event === "click:Approve")).toHaveLength(1);
+  });
+  it("never completes from a pending native recovery after the original campaign deadline", async () => {
+    let now = Date.now();
+    const failedAt = now - 1000, retryAt = now + 1000;
+    vi.spyOn(Date, "now").mockImplementation(() => now);
+    vi.mocked(connectionDelay).mockImplementation(async ms => { now += ms; });
+    const f = fixture({ mutate(phase, state) {
+      if (phase !== "accepted") return;
+      const child = state.runs.find(run => run.id === "child-run")!;
+      withPendingNativeRecovery(child, failedAt, retryAt);
+      child.execution.phase = now < retryAt ? "retry_scheduled" : "reconnecting";
+    } });
+    f.input.deadlineAt = now + 3000;
+    await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_deadline" });
+    expect(f.events).not.toContain("visible:download");
+  });
+  it.each([
+    ["missing projection", (run: Row) => { delete run.execution; }],
+    ["legacy execution", (run: Row) => { run.runtimeMode = "legacy"; }],
+    ["terminal failure", (run: Row) => { run.nativePhase = "terminal_failure"; }],
+    ["missing native issue", (run: Row) => { run.nativeIssueId = null; }],
+    ["mismatched issue binding", (run: Row) => { run.contextSnapshot.issueId = "other-issue"; }],
+    ["provider failure", (run: Row) => { run.errorCode = run.execution.cause = "native_provider_terminal_failed"; }],
+    ["mismatched cause", (run: Row) => { run.execution.cause = "provider_transport_failed"; }],
+    ["generic recovery", (run: Row) => { run.execution.phase = "recovery_needed"; }],
+    ["board owner", (run: Row) => { run.execution.recoveryOwner = "board"; }],
+    ["missing schedule", (run: Row) => { run.execution.retryAt = null; }],
+    ["malformed schedule", (run: Row) => { run.execution.retryAt = "invalid"; }],
+    ["old workspace schedule", (run: Row) => { run.execution.retryAt = new Date(Date.parse(run.finishedAt) - 1).toISOString(); }],
+    ["out-of-campaign schedule", (run: Row) => { run.execution.retryAt = new Date(Date.now() + 60_001).toISOString(); }],
+    ["exhausted attempts", (run: Row) => { run.execution.attempt = 3; }],
+    ["missing attempt", (run: Row) => { delete run.execution.attempt; }],
+    ["noninteger attempt", (run: Row) => { run.execution.attempt = 1.5; }],
+    ["unexpected ceiling", (run: Row) => { run.execution.maxAttempts = 4; }],
+    ["different successor", (run: Row) => { run.execution.successorRunId = "other-run"; }],
+    ["missing successor state", (run: Row) => { delete run.execution.successorRunId; }],
+  ] as const)("keeps a failed native run fatal with %s", async (_label, change) => {
+    const now = Date.now();
+    vi.spyOn(Date, "now").mockReturnValue(now);
+    const f = fixture({ mutate(phase, state) {
+      if (phase !== "accepted") return;
+      const child = state.runs.find(run => run.id === "child-run")!;
+      withPendingNativeRecovery(child, now - 1000, now + 30_000); change(child);
+    } });
+    await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" });
+    expect(f.events).not.toContain("visible:download");
+  });
   it.each(["transient_failure", "ai_connection_busy", "ai_connection_pool_wait", "max_turns_continuation", "issue_disposition_repair"])("starts the workspace wait counter independently of %s", async reason => {
     const f = fixture({ mutate(phase, state) {
       if (phase !== "accepted") return;
@@ -438,3 +525,11 @@ function workspaceDeferral(): Row {
       cancellation: { source: "control_plane", expected: true, initiator: { type: "system" } },
     } };
 }
+
+function withPendingNativeRecovery(run: Row, failedAt: number, retryAt: number) {
+  Object.assign(run, { status: "failed", nativePhase: "retryable_failure", errorCode: "native_session_interrupted",
+    finishedAt: new Date(failedAt).toISOString(), execution: {
+      phase: "retry_scheduled", cause: "native_session_interrupted", retryAt: new Date(retryAt).toISOString(),
+      attempt: 1, maxAttempts: 3, recoveryOwner: "agent", successorRunId: null,
+    } });
+}

tests/runner-e2e/product-journeys/onboarding-first-task.ts

@@ -7,7 +7,7 @@ import { isBlockedUnstartedWake, isTerminalUnstartedWake } from "../non-executio
 import { JourneyError, type JourneyContext, type OnboardingArtifactEvidence, type TargetSession } from "./types.js";
 import { retainOnboardingArtifact } from "./artifact-evidence.js";
 import { preAcceptanceTaskStatus, preAcceptanceViolations, type OnboardingPreAcceptanceEvidence } from "./onboarding-pre-acceptance.js";
-import { isProviderUnstartedWorkspaceDeferral } from "./workspace-deferral.js";
+import { isPendingNativeRecovery, isProviderUnstartedWorkspaceDeferral } from "./workspace-deferral.js";
 
 export interface OnboardingFirstTaskInput {
   context: JourneyContext;
@@ -41,7 +41,6 @@ function fail(code: string): never { throw new JourneyError("failed", code); }
 const runIssueId = (run: Row) => run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId;
 const isProposal = (row: Row) => ["request_confirmation", "request_checkbox_confirmation"].includes(row.kind);
 const isPlanningDocument = (row: Row) => /^(?:plan|proposal)$/.test(row.key);
-const isPendingRun = (run: Row) => activeRuns([run]).length > 0 || run.status === "scheduled_retry";
 
 function sameWorkspaceRetry(source: Row, successor: Row): boolean {
   const issueIds = (run: Row) => [run.nativeIssueId, run.issueId, run.contextSnapshot?.issueId].filter(value => value != null);
@@ -65,6 +64,8 @@ function unstartedScheduledWorkspaceRetry(run: Row): boolean {
 /** Exercise the ordinary onboarding conversation; APIs observe, never submit its actions. */
 export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Promise<OnboardingFirstTaskResult> {
   const { context: { api, page, company }, agent, campaignId, proof } = input;
+  const isPendingRun = (run: Row) => activeRuns([run]).length > 0 || run.status === "scheduled_retry" ||
+    isPendingNativeRecovery(run, input.deadlineAt);
   if (!Number.isSafeInteger(input.maxRuns) || input.maxRuns < 1) fail("invalid_onboarding_run_limit");
   const title = `Artifact ${campaignId}`;
   const seen = new Map<string, Row>();
@@ -136,6 +137,7 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom
         workspacePending = true;
         continue;
       }
+      if (isPendingNativeRecovery(run, input.deadlineAt)) { workspacePending = true; continue; }
       if (activeRuns([run]).length || run.status === "succeeded" || isBlockedUnstartedWake(run) || isTerminalUnstartedWake(run, snapshot.tasks)) continue;
       fail("onboarding_first_task_failed");
     }

tests/runner-e2e/product-journeys/workspace-deferral.ts

@@ -15,6 +15,21 @@ function hasProviderUsage(value: unknown): boolean {
   });
 }
 
+export function isPendingNativeRecovery(run: RecordValue, deadlineAt: number): boolean {
+  const execution = object(run.execution);
+  const finishedAt = typeof run.finishedAt === "string" ? Date.parse(run.finishedAt) : NaN;
+  const retryAt = typeof execution.retryAt === "string" ? Date.parse(execution.retryAt) : NaN;
+  return run.runtimeMode === "native" && run.status === "failed" && run.nativePhase === "retryable_failure" &&
+    run.errorCode === "native_session_interrupted" && execution.cause === "native_session_interrupted" &&
+    typeof run.nativeIssueId === "string" && run.nativeIssueId === object(run.contextSnapshot).issueId &&
+    (run.issueId == null || run.issueId === run.nativeIssueId) &&
+    (execution.phase === "retry_scheduled" || execution.phase === "reconnecting") && execution.recoveryOwner === "agent" &&
+    Number.isInteger(execution.attempt) && Number(execution.attempt) >= 1 &&
+    execution.maxAttempts === 3 && Number(execution.attempt) < execution.maxAttempts && execution.successorRunId === null &&
+    Number.isFinite(finishedAt) && Number.isFinite(retryAt) && Number.isFinite(deadlineAt) &&
+    retryAt > finishedAt && retryAt <= deadlineAt;
+}
+
 /** Admission may set startedAt before selecting a runtime. Only this explicit
  * control-plane receipt, without contradictory provider evidence, proves no turn
  * ran. A legacy default alone is never proof of either execution or nonexecution. */