test: isolate native continuation from browser reconnection

Use the production task-comment API only for the explicit post-completion controller replacement follow-up. Preserve the five-minute active outage and strict native/provider identity, tool, saved-project output, and finalization checks. Retain definition 6 browser navigation failure and label post-C API evidence without rendered-status claims.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/EVERYDAY-WORKFLOWS.md

@@ -114,6 +114,13 @@ effects with different tool IDs cannot repeat.
 `remote-restart-before.json`, `remote-restart-after.json` and
 `remote-restart-final.json` retain receipt identities and hashes. They exclude
 provider text, tool arguments, command output, and raw provider sandbox IDs.
+The dedicated explicit-only `daytona-restart-continuity` definition 7 additionally
+replaces the controller after completion, then posts its unchanged saved-project
+follow-up through the normal authenticated task-comment API. Exactly one matching
+persisted user comment, consumed wake, same native/provider identity, new completed
+tool and useful output are required. Its initial browser journey remains; post-C
+readback is API-only and does not qualify browser reconnection or rendered status.
+The definition 6 browser timeout remains failed with an unknown browser cause.
 The suite definition version and harness digest change with this oracle.
 Its deterministic calibration is not a claim that a live Daytona run passed.
 

tests/runner-e2e/README.md

@@ -29,8 +29,17 @@ ownership/reconnect limits, archive/delete backstops and owned cleanup remain un
 The retained continuation accepts whitespace between its exact marker and the
 exact `queue_r` output token; readiness and final grading use the same matcher.
 
-Definition version 6 uses the existing warm reusable environment with a dedicated
-configuration key. The prior version 5 per-turn/no-reuse measurement **failed**
+Definition version 7 uses the existing warm reusable environment with a dedicated
+configuration key. Only the post-completion replacement follow-up uses the normal
+board-authenticated task-comment API, with one persisted user comment and the
+unchanged follow-up request. Initial creation, active outage and queued continuation
+retain their browser journey. Post-C settlement, task status and useful reply are
+read through production APIs; no post-C browser navigation, rendered status or
+final screenshot is claimed. The same strict native/provider identity, new tool,
+saved-project output and finalization checks remain required. Definition 6's
+observed post-C browser navigation timeout remains failed; its cause is unknown.
+This definition does not claim that browser reconnection was fixed.
+The prior version 5 per-turn/no-reuse measurement **failed**
 retained native/provider identity and remains failed; it does not qualify this
 warm fixture. Cache retirement moves the entire controller Runner directory,
 including provider failover-backup bytes. Once a per-turn sandbox is disposed,

tests/runner-e2e/catalog.test.ts

@@ -70,8 +70,8 @@ describe("runner E2E catalog", () => {
     }
     expect(baseConfig).toMatchObject({ autoStopInterval: 5, reuseLease: false, runnerLifecycleMode: "per_turn" });
     expect(warmConfig).toMatchObject({ autoStopInterval: 5, reuseLease: true, runnerLifecycleMode: "warm" });
-    expect(suite.definitionMetadata).toMatchObject({ version: 6, controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15,
-      retainedControllerContinuation: true, maximumAttemptsPerCell: 1 });
+    expect(suite.definitionMetadata).toMatchObject({ version: 7, controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15,
+      retainedControllerContinuation: true, retainedControllerFollowupTransport: "task_comment_api", maximumAttemptsPerCell: 1 });
   });
 
   it.each([

tests/runner-e2e/catalog.ts

@@ -1173,7 +1173,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
     }],
     tasks: everydayTasks.filter(task => task.id === "recover-controller")
       .map(task => ({ ...task, expectedRunCount: task.expectedRunCount + 1, automaticRetryPolicy: "single_attempt" as const })), expectedMatrixSize: 2,
-    definitionMetadata: { version: 6, scheduling: "explicit-only", controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, retainedControllerContinuation: true,
+    definitionMetadata: { version: 7, scheduling: "explicit-only", controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, retainedControllerContinuation: true, retainedControllerFollowupTransport: "task_comment_api",
       maximumAttemptsPerCell: 1, budgetMonthlyCents: 1000, instructions: "production", grading: "same-active-process-lease-native-prefix-and-workspace-receipt",
       authentication: "api-key", stagingAuthentication: "subscription", stagingQualification: false },
   },

tests/runner-e2e/everyday-flow.ts

@@ -8,7 +8,7 @@ import { gradeConnectionGuidanceDecline, hasConnectionGuidanceDeclineReply, expl
 import { expect, type Page } from "@playwright/test";
 import { runnerApiToolsEnabled } from "../../server/src/services/native-runtime/runner-api-rollout.js";
 import { spawn } from "node:child_process";
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { mkdir, readFile, readdir } from "node:fs/promises";
 import path from "node:path";
 import { isDeepStrictEqual } from "node:util";
@@ -26,7 +26,7 @@ import { hasPersistedSource, isSavedSourceCheckpoint } from "./everyday-interrup
 import { collectRunEvents } from "./run-observations.js";
 import { gradeRemoteRestart, observeRemoteRestartBoundary, projectRemoteRestartEvents, projectRemoteRestartRun, waitForRemoteRestartBoundary, type RemoteRestartEvidence } from "./everyday-remote-restart.js";
 import { controllerOutageForExecution, type ControllerOutageReceipt } from "./controller-outage.js";
-import { gradeRetainedControllerContinuation, retainedControllerRun } from "./retained-controller-continuation.js";
+import { gradeRetainedControllerContinuation, retainedControllerRun, retainedControllerFollowupTransport, submitRetainedControllerComment, onePersistedRetainedControllerComment } from "./retained-controller-continuation.js";
 import { setupAggregatorFixture } from "./aggregator-fixture.js";
 import { gradeProviderChoice, gradeProviderOutcome, requireProviderAccessCard } from "./connection-routing-evidence.js";
 import { setupConnectionReview } from "./connection-reviews.js";
@@ -191,6 +191,8 @@ export async function runEverydayFlow(input: Input) {
   let providerAccessDecision: { id: string; connectionId: string } | undefined;
   let stoppedWorkspace: Record<string, string> | undefined;
   let settledAgentReply: Row | undefined;
+  let postControllerApiFollowup = false;
+  let retainedFollowupCommentId: string | undefined;
   let reviewHandoffBoundary: {
     childId: string;
     assigneeAgentId: string | null | undefined;
@@ -284,7 +286,7 @@ export async function runEverydayFlow(input: Input) {
       ),
     ];
   }
-  async function reply(message: string, target: StoryIssue = parent!) {
+  async function reply(message: string, target: StoryIssue = parent!, transport: "browser" | "task_comment_api" = "browser") {
     const priorFailures = ev.checks.filter((c) => !c.passed);
     if (priorFailures.length)
       throw new Error(
@@ -292,7 +294,9 @@ export async function runEverydayFlow(input: Input) {
       );
     const before = await api.get<Row[]>(`/api/issues/${target.id}/comments`);
     lastSubmissionAt = Date.now();
-    await submitTaskReply(page, message);
+    const submitted = transport === "task_comment_api"
+      ? await submitRetainedControllerComment(api, target.id, message, randomUUID())
+      : (await submitTaskReply(page, message), undefined);
     const after = await pollUntil({
       label: "one persisted user reply",
       deadlineAt: Date.now() + 30_000,
@@ -307,11 +311,12 @@ export async function runEverydayFlow(input: Input) {
     );
     check(
       `reply-${ev.timeline.length}-stored-once`,
-      added.length === 1,
-      "A composer submission creates exactly one user message.",
+      transport === "task_comment_api" ? onePersistedRetainedControllerComment({ added, commentId: submitted?.id ?? "", body: message }) : added.length === 1,
+      transport === "task_comment_api" ? "The production comment API creates exactly one matching user message." : "A composer submission creates exactly one user message.",
     );
     submittedCommentIds.push(...added.map((c) => c.id));
-    note("composer-message-persisted", {
+    if (submitted) retainedFollowupCommentId = submitted.id;
+    note(transport === "task_comment_api" ? "task-api-message-persisted" : "composer-message-persisted", {
       issueId: target.id,
       commentIds: added.map((c) => c.id),
     });
@@ -328,7 +333,7 @@ export async function runEverydayFlow(input: Input) {
       sameConnections: false,
     };
   }
-  async function settled(expectedAgentReply?: StoryReplyExpectation) {
+  async function settled(expectedAgentReply?: StoryReplyExpectation, readback: "browser" | "api" = "browser") {
     const settledState = await pollUntil({
       label: `everyday ${caseId} settled`,
       deadlineAt: input.deadlineAt,
@@ -400,7 +405,7 @@ export async function runEverydayFlow(input: Input) {
         // later refresh may return a newer projection and must not change the
         // evidence used by the assertion below.
         settledAgentReply = { ...reply };
-        note("expected-agent-reply-visible-at-settlement", {
+        note(readback === "api" ? "expected-agent-reply-persisted-at-settlement" : "expected-agent-reply-visible-at-settlement", {
           issueId: issue?.id,
           commentId: reply.id,
           authorAgentId: reply.authorAgentId,
@@ -409,14 +414,16 @@ export async function runEverydayFlow(input: Input) {
         });
       }
     }
-    await openParent();
-    await expect(
-      page.getByTestId("issue-detail-header").getByRole("button", {
-        name: "Change status (current: Done)",
-        exact: true,
-      }),
-    ).toBeVisible();
-    note("all-tasks-done");
+    if (readback === "browser") {
+      await openParent();
+      await expect(
+        page.getByTestId("issue-detail-header").getByRole("button", {
+          name: "Change status (current: Done)",
+          exact: true,
+        }),
+      ).toBeVisible();
+    }
+    note("all-tasks-done", { readback });
   }
   async function downloadDelegated(
     mode: "base" | "separator" | "max-length",
@@ -576,6 +583,7 @@ export async function runEverydayFlow(input: Input) {
       "everyday-delivery.ts",
       "everyday-observations.ts",
       "everyday-remote-restart.ts",
+      "retained-controller-continuation.ts",
       "controller-outage.ts",
       "controller-recovery-observer.ts",
       "upgrade-artifacts.ts",
@@ -1454,16 +1462,21 @@ export async function runEverydayFlow(input: Input) {
         if (!prior) throw new Error("Retained continuation requires a completed source run");
         const before = retainedControllerRun(prior), priorIds = new Set(ev.runs.map(run => run.id));
         const outage = await input.restart();
-        await openParent();
+        const followupTransport = retainedControllerFollowupTransport(execution);
+        postControllerApiFollowup = followupTransport === "task_comment_api";
+        if (!postControllerApiFollowup) await openParent();
+        note("retained-controller-followup-entry", { transport: followupTransport,
+          browserNavigationVerified: !postControllerApiFollowup });
         const requestedAt = new Date().toISOString(), marker = `CONTINUED_${nonce}`;
         const expectedReply = retainedControllerReplyMatcher(marker);
-        await reply(`Run the existing delivered slugify.py on "  Queue--Ready!!  " with --separator _ and --max-length 7. Report its output after the marker ${marker}. Use the saved project; do not rebuild it.`);
-        await settled(expectedReply);
+        await reply(`Run the existing delivered slugify.py on "  Queue--Ready!!  " with --separator _ and --max-length 7. Report its output after the marker ${marker}. Use the saved project; do not rebuild it.`, parent!, followupTransport);
+        await settled(expectedReply, postControllerApiFollowup ? "api" : "browser");
         const after = ev.runs.filter(run => !priorIds.has(run.id) && run.status === "succeeded" && run.agentId === fixtures.agent.id)
           .sort((a, b) => Date.parse(b.finishedAt ?? "") - Date.parse(a.finishedAt ?? ""))[0];
         if (!after) throw new Error("Retained continuation produced no new successful run");
         const events = await collectRunEvents<Row>((afterSeq, limit) => api.get(`/api/heartbeat-runs/${after.id}/events?afterSeq=${afterSeq}&limit=${limit}`));
         const evidence = { before, after: retainedControllerRun(after), outage: outage || undefined, requestedAt,
+          followupTransport, followupCommentId: retainedFollowupCommentId,
           completedTool: events.some(row => row.payload?.prpEvent?.eventType === "tool.execution.completed" && row.payload.prpEvent.payload?.status === "completed"),
           usefulReply: storyAgentReplyMatches(settledAgentReply, fixtures.agent.id, expectedReply) };
         ev.checks.push(...gradeRetainedControllerContinuation(evidence));
@@ -1503,7 +1516,7 @@ export async function runEverydayFlow(input: Input) {
         ev.issues.find((i) => i.id === parent!.id)!.documents = bodies;
       }
     }
-    await page.reload();
+    if (!postControllerApiFollowup) await page.reload();
     await refresh();
     check(
       "submitted-replies-consumed",
@@ -1598,25 +1611,27 @@ export async function runEverydayFlow(input: Input) {
       ),
       "No completion confirmation or unanswered interaction remains.",
     );
-    if (providerChoice || nativeProviderCase) await openParent();
-    else {
-      await waitForTaskIdentity(page, taskUrl(parent!), parent!.identifier ?? parent!.id);
-      await waitForTaskChatRendered(page);
-    }
-    const latestAgentComment = ev.issues
-      .find((i) => i.id === parent!.id)
-      ?.comments?.filter((c: Row) => c.authorAgentId)
-      .at(-1);
-    if (latestAgentComment && !providerChoice && !nativeProviderCase) {
-      const response = page.locator(`[id="comment-${latestAgentComment.id}"]`);
-      await expect(response).toBeVisible();
-      await response.scrollIntoViewIfNeeded();
-    }
-    await input.capture(
-      "final-state",
-      "Finished everyday workflow",
-      "final-state.png",
-    );
+    if (!postControllerApiFollowup) {
+      if (providerChoice || nativeProviderCase) await openParent();
+      else {
+        await waitForTaskIdentity(page, taskUrl(parent!), parent!.identifier ?? parent!.id);
+        await waitForTaskChatRendered(page);
+      }
+      const latestAgentComment = ev.issues
+        .find((i) => i.id === parent!.id)
+        ?.comments?.filter((c: Row) => c.authorAgentId)
+        .at(-1);
+      if (latestAgentComment && !providerChoice && !nativeProviderCase) {
+        const response = page.locator(`[id="comment-${latestAgentComment.id}"]`);
+        await expect(response).toBeVisible();
+        await response.scrollIntoViewIfNeeded();
+      }
+      await input.capture(
+        "final-state",
+        "Finished everyday workflow",
+        "final-state.png",
+      );
+    } else note("final-state-api-readback", { browserNavigationVerified: false, renderedStatusVerified: false });
     const failed = ev.checks.filter((c) => !c.passed);
     if (failed.length)
       throw new Error(

tests/runner-e2e/retained-controller-continuation.test.ts

@@ -1,5 +1,7 @@
-import { describe, expect, it } from "vitest";
-import { gradeRetainedControllerContinuation, retainedControllerRun } from "./retained-controller-continuation.js";
+import { describe, expect, it, vi } from "vitest";
+import { gradeRetainedControllerContinuation, retainedControllerRun, retainedControllerFollowupTransport, submitRetainedControllerComment, onePersistedRetainedControllerComment } from "./retained-controller-continuation.js";
+
+import { runnerMatrix } from "./catalog.js";
 
 function fixture() {
   const before = retainedControllerRun({ id: "a", companyId: "company", agentId: "agent", nativeIssueId: "task",
@@ -29,3 +31,35 @@ describe("retained controller continuation evidence", () => {
     expect(gradeRetainedControllerContinuation(input)[0]?.passed).toBe(false);
   });
 });
+
+describe("post-controller API follow-up selection", () => {
+  const execution = runnerMatrix.find(cell => cell.id === "daytona-restart-continuity.runner-codex.daytona.recover-controller")!;
+  it("selects only the explicit dedicated definition and defaults to the browser", () => {
+    expect(retainedControllerFollowupTransport(execution)).toBe("task_comment_api");
+    for (const changed of [
+      { ...execution, suite: { ...execution.suite, id: "everyday-workflows" } },
+      { ...execution, task: { ...execution.task, id: "another-case" } },
+      { ...execution, suite: { ...execution.suite, definitionMetadata: { retainedControllerContinuation: true } } },
+      { ...execution, suite: { ...execution.suite, definitionMetadata: { retainedControllerFollowupTransport: "task_comment_api" } } },
+    ]) expect(retainedControllerFollowupTransport(changed)).toBe("browser");
+  });
+  it("posts one ordinary authenticated comment request with a stable client request ID", async () => {
+    const post = vi.fn(async () => ({ id: "saved-user-comment" }));
+    const api = { post } as unknown as Parameters<typeof submitRetainedControllerComment>[0];
+    await expect(submitRetainedControllerComment(api, "task-id", "Exact original follow-up.", "request-id"))
+      .resolves.toEqual({ id: "saved-user-comment" });
+    expect(post).toHaveBeenCalledExactlyOnceWith("/api/issues/task-id/comments", {
+      body: "Exact original follow-up.", reopen: true, clientRequestId: "request-id",
+    });
+  });
+  it("requires exactly the matching persisted user comment", () => {
+    const comment = { id: "comment", body: "Exact follow-up", authorType: "user" };
+    const proof = { added: [comment], commentId: comment.id, body: comment.body };
+    expect(onePersistedRetainedControllerComment(proof)).toBe(true);
+    expect(onePersistedRetainedControllerComment({ ...proof, commentId: "" })).toBe(false);
+    for (const added of [[], [comment, { ...comment, id: "duplicate" }], [{ ...comment, id: "other" }],
+      [{ ...comment, body: "changed" }], [{ ...comment, authorType: "system" }], [{ ...comment, authorAgentId: "agent" }]]) {
+      expect(onePersistedRetainedControllerComment({ ...proof, added })).toBe(false);
+    }
+  });
+});

tests/runner-e2e/retained-controller-continuation.ts

@@ -1,7 +1,30 @@
+import type { RunnerApi } from "./api.js";
+import type { MatrixExecution } from "./types.js";
 import { createHash } from "node:crypto";
 import type { StoryCheck, StoryRun } from "./everyday-observations.js";
 import { validControllerOutageReceipt, type ControllerOutageReceipt } from "./controller-outage.js";
 
+/** Only this explicit definition changes the post-replacement user entry point. */
+export function retainedControllerFollowupTransport(execution: Pick<MatrixExecution, "suite" | "task">): "browser" | "task_comment_api" {
+  return execution.suite.id === "daytona-restart-continuity" && execution.task.id === "recover-controller"
+    && execution.suite.definitionMetadata?.retainedControllerContinuation === true
+    && execution.suite.definitionMetadata.retainedControllerFollowupTransport === "task_comment_api"
+    ? "task_comment_api" : "browser";
+}
+
+export async function submitRetainedControllerComment(api: Pick<RunnerApi, "post">, issueId: string, body: string, clientRequestId: string) {
+  return api.post<{ id: string }>(`/api/issues/${issueId}/comments`, { body, reopen: true, clientRequestId });
+}
+
+export function onePersistedRetainedControllerComment(input: {
+  added: Array<{ id?: unknown; body?: unknown; authorAgentId?: unknown; authorType?: unknown }>;
+  commentId: string; body: string;
+}) {
+  return typeof input.commentId === "string" && input.commentId.length > 0
+    && input.added.length === 1 && input.added[0]?.id === input.commentId
+    && input.added[0].body === input.body && !input.added[0].authorAgentId && input.added[0].authorType === "user";
+}
+
 const hash = (value: string | null | undefined) => value ? createHash("sha256").update(value).digest("hex") : null;
 
 /** Public evidence keeps provider/session identifiers hashed. */