From 4e8e83c5178af548ba17a082367269cbd9f22e0b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:53:41 -0500 Subject: [PATCH] test: isolate native continuation from browser reconnection Use the production task-comment API only for the explicit post-completion controller replacement follow-up. Preserve the five-minute active outage and strict native/provider identity, tool, saved-project output, and finalization checks. Retain definition 6 browser navigation failure and label post-C API evidence without rendered-status claims. Co-Authored-By: Paperclip --- tests/runner-e2e/EVERYDAY-WORKFLOWS.md | 7 ++ tests/runner-e2e/README.md | 13 ++- tests/runner-e2e/catalog.test.ts | 4 +- tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/everyday-flow.ts | 95 +++++++++++-------- .../retained-controller-continuation.test.ts | 38 +++++++- .../retained-controller-continuation.ts | 23 +++++ 7 files changed, 135 insertions(+), 47 deletions(-) diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index dbb11f6b884..1d7bf6f6fcb 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -114,6 +114,13 @@ effects with different tool IDs cannot repeat. `remote-restart-before.json`, `remote-restart-after.json` and `remote-restart-final.json` retain receipt identities and hashes. They exclude provider text, tool arguments, command output, and raw provider sandbox IDs. +The dedicated explicit-only `daytona-restart-continuity` definition 7 additionally +replaces the controller after completion, then posts its unchanged saved-project +follow-up through the normal authenticated task-comment API. Exactly one matching +persisted user comment, consumed wake, same native/provider identity, new completed +tool and useful output are required. Its initial browser journey remains; post-C +readback is API-only and does not qualify browser reconnection or rendered status. +The definition 6 browser timeout remains failed with an unknown browser cause. The suite definition version and harness digest change with this oracle. Its deterministic calibration is not a claim that a live Daytona run passed. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 1107bc6ab18..e42c5cce5a4 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -29,8 +29,17 @@ ownership/reconnect limits, archive/delete backstops and owned cleanup remain un The retained continuation accepts whitespace between its exact marker and the exact `queue_r` output token; readiness and final grading use the same matcher. -Definition version 6 uses the existing warm reusable environment with a dedicated -configuration key. The prior version 5 per-turn/no-reuse measurement **failed** +Definition version 7 uses the existing warm reusable environment with a dedicated +configuration key. Only the post-completion replacement follow-up uses the normal +board-authenticated task-comment API, with one persisted user comment and the +unchanged follow-up request. Initial creation, active outage and queued continuation +retain their browser journey. Post-C settlement, task status and useful reply are +read through production APIs; no post-C browser navigation, rendered status or +final screenshot is claimed. The same strict native/provider identity, new tool, +saved-project output and finalization checks remain required. Definition 6's +observed post-C browser navigation timeout remains failed; its cause is unknown. +This definition does not claim that browser reconnection was fixed. +The prior version 5 per-turn/no-reuse measurement **failed** retained native/provider identity and remains failed; it does not qualify this warm fixture. Cache retirement moves the entire controller Runner directory, including provider failover-backup bytes. Once a per-turn sandbox is disposed, diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 3bcb8da7b91..f89c17674e0 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -70,8 +70,8 @@ describe("runner E2E catalog", () => { } expect(baseConfig).toMatchObject({ autoStopInterval: 5, reuseLease: false, runnerLifecycleMode: "per_turn" }); expect(warmConfig).toMatchObject({ autoStopInterval: 5, reuseLease: true, runnerLifecycleMode: "warm" }); - expect(suite.definitionMetadata).toMatchObject({ version: 6, controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, - retainedControllerContinuation: true, maximumAttemptsPerCell: 1 }); + expect(suite.definitionMetadata).toMatchObject({ version: 7, controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, + retainedControllerContinuation: true, retainedControllerFollowupTransport: "task_comment_api", maximumAttemptsPerCell: 1 }); }); it.each([ diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 485324102de..7c1dca62b36 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1173,7 +1173,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ }], tasks: everydayTasks.filter(task => task.id === "recover-controller") .map(task => ({ ...task, expectedRunCount: task.expectedRunCount + 1, automaticRetryPolicy: "single_attempt" as const })), expectedMatrixSize: 2, - definitionMetadata: { version: 6, scheduling: "explicit-only", controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, retainedControllerContinuation: true, + definitionMetadata: { version: 7, scheduling: "explicit-only", controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, retainedControllerContinuation: true, retainedControllerFollowupTransport: "task_comment_api", maximumAttemptsPerCell: 1, budgetMonthlyCents: 1000, instructions: "production", grading: "same-active-process-lease-native-prefix-and-workspace-receipt", authentication: "api-key", stagingAuthentication: "subscription", stagingQualification: false }, }, diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index 649b3217d8a..546a4031c46 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -8,7 +8,7 @@ import { gradeConnectionGuidanceDecline, hasConnectionGuidanceDeclineReply, expl import { expect, type Page } from "@playwright/test"; import { runnerApiToolsEnabled } from "../../server/src/services/native-runtime/runner-api-rollout.js"; import { spawn } from "node:child_process"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { mkdir, readFile, readdir } from "node:fs/promises"; import path from "node:path"; import { isDeepStrictEqual } from "node:util"; @@ -26,7 +26,7 @@ import { hasPersistedSource, isSavedSourceCheckpoint } from "./everyday-interrup import { collectRunEvents } from "./run-observations.js"; import { gradeRemoteRestart, observeRemoteRestartBoundary, projectRemoteRestartEvents, projectRemoteRestartRun, waitForRemoteRestartBoundary, type RemoteRestartEvidence } from "./everyday-remote-restart.js"; import { controllerOutageForExecution, type ControllerOutageReceipt } from "./controller-outage.js"; -import { gradeRetainedControllerContinuation, retainedControllerRun } from "./retained-controller-continuation.js"; +import { gradeRetainedControllerContinuation, retainedControllerRun, retainedControllerFollowupTransport, submitRetainedControllerComment, onePersistedRetainedControllerComment } from "./retained-controller-continuation.js"; import { setupAggregatorFixture } from "./aggregator-fixture.js"; import { gradeProviderChoice, gradeProviderOutcome, requireProviderAccessCard } from "./connection-routing-evidence.js"; import { setupConnectionReview } from "./connection-reviews.js"; @@ -191,6 +191,8 @@ export async function runEverydayFlow(input: Input) { let providerAccessDecision: { id: string; connectionId: string } | undefined; let stoppedWorkspace: Record | undefined; let settledAgentReply: Row | undefined; + let postControllerApiFollowup = false; + let retainedFollowupCommentId: string | undefined; let reviewHandoffBoundary: { childId: string; assigneeAgentId: string | null | undefined; @@ -284,7 +286,7 @@ export async function runEverydayFlow(input: Input) { ), ]; } - async function reply(message: string, target: StoryIssue = parent!) { + async function reply(message: string, target: StoryIssue = parent!, transport: "browser" | "task_comment_api" = "browser") { const priorFailures = ev.checks.filter((c) => !c.passed); if (priorFailures.length) throw new Error( @@ -292,7 +294,9 @@ export async function runEverydayFlow(input: Input) { ); const before = await api.get(`/api/issues/${target.id}/comments`); lastSubmissionAt = Date.now(); - await submitTaskReply(page, message); + const submitted = transport === "task_comment_api" + ? await submitRetainedControllerComment(api, target.id, message, randomUUID()) + : (await submitTaskReply(page, message), undefined); const after = await pollUntil({ label: "one persisted user reply", deadlineAt: Date.now() + 30_000, @@ -307,11 +311,12 @@ export async function runEverydayFlow(input: Input) { ); check( `reply-${ev.timeline.length}-stored-once`, - added.length === 1, - "A composer submission creates exactly one user message.", + transport === "task_comment_api" ? onePersistedRetainedControllerComment({ added, commentId: submitted?.id ?? "", body: message }) : added.length === 1, + transport === "task_comment_api" ? "The production comment API creates exactly one matching user message." : "A composer submission creates exactly one user message.", ); submittedCommentIds.push(...added.map((c) => c.id)); - note("composer-message-persisted", { + if (submitted) retainedFollowupCommentId = submitted.id; + note(transport === "task_comment_api" ? "task-api-message-persisted" : "composer-message-persisted", { issueId: target.id, commentIds: added.map((c) => c.id), }); @@ -328,7 +333,7 @@ export async function runEverydayFlow(input: Input) { sameConnections: false, }; } - async function settled(expectedAgentReply?: StoryReplyExpectation) { + async function settled(expectedAgentReply?: StoryReplyExpectation, readback: "browser" | "api" = "browser") { const settledState = await pollUntil({ label: `everyday ${caseId} settled`, deadlineAt: input.deadlineAt, @@ -400,7 +405,7 @@ export async function runEverydayFlow(input: Input) { // later refresh may return a newer projection and must not change the // evidence used by the assertion below. settledAgentReply = { ...reply }; - note("expected-agent-reply-visible-at-settlement", { + note(readback === "api" ? "expected-agent-reply-persisted-at-settlement" : "expected-agent-reply-visible-at-settlement", { issueId: issue?.id, commentId: reply.id, authorAgentId: reply.authorAgentId, @@ -409,14 +414,16 @@ export async function runEverydayFlow(input: Input) { }); } } - await openParent(); - await expect( - page.getByTestId("issue-detail-header").getByRole("button", { - name: "Change status (current: Done)", - exact: true, - }), - ).toBeVisible(); - note("all-tasks-done"); + if (readback === "browser") { + await openParent(); + await expect( + page.getByTestId("issue-detail-header").getByRole("button", { + name: "Change status (current: Done)", + exact: true, + }), + ).toBeVisible(); + } + note("all-tasks-done", { readback }); } async function downloadDelegated( mode: "base" | "separator" | "max-length", @@ -576,6 +583,7 @@ export async function runEverydayFlow(input: Input) { "everyday-delivery.ts", "everyday-observations.ts", "everyday-remote-restart.ts", + "retained-controller-continuation.ts", "controller-outage.ts", "controller-recovery-observer.ts", "upgrade-artifacts.ts", @@ -1454,16 +1462,21 @@ export async function runEverydayFlow(input: Input) { if (!prior) throw new Error("Retained continuation requires a completed source run"); const before = retainedControllerRun(prior), priorIds = new Set(ev.runs.map(run => run.id)); const outage = await input.restart(); - await openParent(); + const followupTransport = retainedControllerFollowupTransport(execution); + postControllerApiFollowup = followupTransport === "task_comment_api"; + if (!postControllerApiFollowup) await openParent(); + note("retained-controller-followup-entry", { transport: followupTransport, + browserNavigationVerified: !postControllerApiFollowup }); const requestedAt = new Date().toISOString(), marker = `CONTINUED_${nonce}`; const expectedReply = retainedControllerReplyMatcher(marker); - await reply(`Run the existing delivered slugify.py on " Queue--Ready!! " with --separator _ and --max-length 7. Report its output after the marker ${marker}. Use the saved project; do not rebuild it.`); - await settled(expectedReply); + await reply(`Run the existing delivered slugify.py on " Queue--Ready!! " with --separator _ and --max-length 7. Report its output after the marker ${marker}. Use the saved project; do not rebuild it.`, parent!, followupTransport); + await settled(expectedReply, postControllerApiFollowup ? "api" : "browser"); const after = ev.runs.filter(run => !priorIds.has(run.id) && run.status === "succeeded" && run.agentId === fixtures.agent.id) .sort((a, b) => Date.parse(b.finishedAt ?? "") - Date.parse(a.finishedAt ?? ""))[0]; if (!after) throw new Error("Retained continuation produced no new successful run"); const events = await collectRunEvents((afterSeq, limit) => api.get(`/api/heartbeat-runs/${after.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); const evidence = { before, after: retainedControllerRun(after), outage: outage || undefined, requestedAt, + followupTransport, followupCommentId: retainedFollowupCommentId, completedTool: events.some(row => row.payload?.prpEvent?.eventType === "tool.execution.completed" && row.payload.prpEvent.payload?.status === "completed"), usefulReply: storyAgentReplyMatches(settledAgentReply, fixtures.agent.id, expectedReply) }; ev.checks.push(...gradeRetainedControllerContinuation(evidence)); @@ -1503,7 +1516,7 @@ export async function runEverydayFlow(input: Input) { ev.issues.find((i) => i.id === parent!.id)!.documents = bodies; } } - await page.reload(); + if (!postControllerApiFollowup) await page.reload(); await refresh(); check( "submitted-replies-consumed", @@ -1598,25 +1611,27 @@ export async function runEverydayFlow(input: Input) { ), "No completion confirmation or unanswered interaction remains.", ); - if (providerChoice || nativeProviderCase) await openParent(); - else { - await waitForTaskIdentity(page, taskUrl(parent!), parent!.identifier ?? parent!.id); - await waitForTaskChatRendered(page); - } - const latestAgentComment = ev.issues - .find((i) => i.id === parent!.id) - ?.comments?.filter((c: Row) => c.authorAgentId) - .at(-1); - if (latestAgentComment && !providerChoice && !nativeProviderCase) { - const response = page.locator(`[id="comment-${latestAgentComment.id}"]`); - await expect(response).toBeVisible(); - await response.scrollIntoViewIfNeeded(); - } - await input.capture( - "final-state", - "Finished everyday workflow", - "final-state.png", - ); + if (!postControllerApiFollowup) { + if (providerChoice || nativeProviderCase) await openParent(); + else { + await waitForTaskIdentity(page, taskUrl(parent!), parent!.identifier ?? parent!.id); + await waitForTaskChatRendered(page); + } + const latestAgentComment = ev.issues + .find((i) => i.id === parent!.id) + ?.comments?.filter((c: Row) => c.authorAgentId) + .at(-1); + if (latestAgentComment && !providerChoice && !nativeProviderCase) { + const response = page.locator(`[id="comment-${latestAgentComment.id}"]`); + await expect(response).toBeVisible(); + await response.scrollIntoViewIfNeeded(); + } + await input.capture( + "final-state", + "Finished everyday workflow", + "final-state.png", + ); + } else note("final-state-api-readback", { browserNavigationVerified: false, renderedStatusVerified: false }); const failed = ev.checks.filter((c) => !c.passed); if (failed.length) throw new Error( diff --git a/tests/runner-e2e/retained-controller-continuation.test.ts b/tests/runner-e2e/retained-controller-continuation.test.ts index 76beee4cd42..a9636dfb06e 100644 --- a/tests/runner-e2e/retained-controller-continuation.test.ts +++ b/tests/runner-e2e/retained-controller-continuation.test.ts @@ -1,5 +1,7 @@ -import { describe, expect, it } from "vitest"; -import { gradeRetainedControllerContinuation, retainedControllerRun } from "./retained-controller-continuation.js"; +import { describe, expect, it, vi } from "vitest"; +import { gradeRetainedControllerContinuation, retainedControllerRun, retainedControllerFollowupTransport, submitRetainedControllerComment, onePersistedRetainedControllerComment } from "./retained-controller-continuation.js"; + +import { runnerMatrix } from "./catalog.js"; function fixture() { const before = retainedControllerRun({ id: "a", companyId: "company", agentId: "agent", nativeIssueId: "task", @@ -29,3 +31,35 @@ describe("retained controller continuation evidence", () => { expect(gradeRetainedControllerContinuation(input)[0]?.passed).toBe(false); }); }); + +describe("post-controller API follow-up selection", () => { + const execution = runnerMatrix.find(cell => cell.id === "daytona-restart-continuity.runner-codex.daytona.recover-controller")!; + it("selects only the explicit dedicated definition and defaults to the browser", () => { + expect(retainedControllerFollowupTransport(execution)).toBe("task_comment_api"); + for (const changed of [ + { ...execution, suite: { ...execution.suite, id: "everyday-workflows" } }, + { ...execution, task: { ...execution.task, id: "another-case" } }, + { ...execution, suite: { ...execution.suite, definitionMetadata: { retainedControllerContinuation: true } } }, + { ...execution, suite: { ...execution.suite, definitionMetadata: { retainedControllerFollowupTransport: "task_comment_api" } } }, + ]) expect(retainedControllerFollowupTransport(changed)).toBe("browser"); + }); + it("posts one ordinary authenticated comment request with a stable client request ID", async () => { + const post = vi.fn(async () => ({ id: "saved-user-comment" })); + const api = { post } as unknown as Parameters[0]; + await expect(submitRetainedControllerComment(api, "task-id", "Exact original follow-up.", "request-id")) + .resolves.toEqual({ id: "saved-user-comment" }); + expect(post).toHaveBeenCalledExactlyOnceWith("/api/issues/task-id/comments", { + body: "Exact original follow-up.", reopen: true, clientRequestId: "request-id", + }); + }); + it("requires exactly the matching persisted user comment", () => { + const comment = { id: "comment", body: "Exact follow-up", authorType: "user" }; + const proof = { added: [comment], commentId: comment.id, body: comment.body }; + expect(onePersistedRetainedControllerComment(proof)).toBe(true); + expect(onePersistedRetainedControllerComment({ ...proof, commentId: "" })).toBe(false); + for (const added of [[], [comment, { ...comment, id: "duplicate" }], [{ ...comment, id: "other" }], + [{ ...comment, body: "changed" }], [{ ...comment, authorType: "system" }], [{ ...comment, authorAgentId: "agent" }]]) { + expect(onePersistedRetainedControllerComment({ ...proof, added })).toBe(false); + } + }); +}); diff --git a/tests/runner-e2e/retained-controller-continuation.ts b/tests/runner-e2e/retained-controller-continuation.ts index 0601c5815a3..d0f5fa4bb31 100644 --- a/tests/runner-e2e/retained-controller-continuation.ts +++ b/tests/runner-e2e/retained-controller-continuation.ts @@ -1,7 +1,30 @@ +import type { RunnerApi } from "./api.js"; +import type { MatrixExecution } from "./types.js"; import { createHash } from "node:crypto"; import type { StoryCheck, StoryRun } from "./everyday-observations.js"; import { validControllerOutageReceipt, type ControllerOutageReceipt } from "./controller-outage.js"; +/** Only this explicit definition changes the post-replacement user entry point. */ +export function retainedControllerFollowupTransport(execution: Pick): "browser" | "task_comment_api" { + return execution.suite.id === "daytona-restart-continuity" && execution.task.id === "recover-controller" + && execution.suite.definitionMetadata?.retainedControllerContinuation === true + && execution.suite.definitionMetadata.retainedControllerFollowupTransport === "task_comment_api" + ? "task_comment_api" : "browser"; +} + +export async function submitRetainedControllerComment(api: Pick, issueId: string, body: string, clientRequestId: string) { + return api.post<{ id: string }>(`/api/issues/${issueId}/comments`, { body, reopen: true, clientRequestId }); +} + +export function onePersistedRetainedControllerComment(input: { + added: Array<{ id?: unknown; body?: unknown; authorAgentId?: unknown; authorType?: unknown }>; + commentId: string; body: string; +}) { + return typeof input.commentId === "string" && input.commentId.length > 0 + && input.added.length === 1 && input.added[0]?.id === input.commentId + && input.added[0].body === input.body && !input.added[0].authorAgentId && input.added[0].authorType === "user"; +} + const hash = (value: string | null | undefined) => value ? createHash("sha256").update(value).digest("hex") : null; /** Public evidence keeps provider/session identifiers hashed. */