From af9e2d157dcc349e55374ad186c9be2d6cb96bf9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:10:23 -0500 Subject: [PATCH] test: match native continuation replies across whitespace Use one explicit author-bound reply predicate for continuation readiness and evidence. Preserve literal matching elsewhere and reject wrong markers and output tokens. Keep original failed live measurements unchanged. Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 2 ++ tests/runner-e2e/catalog.test.ts | 2 +- tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/everyday-flow.ts | 14 +++++---- tests/runner-e2e/everyday-observations.ts | 25 ++++++++++++--- tests/runner-e2e/everyday.test.ts | 38 +++++++++++++++++++++++ 6 files changed, 71 insertions(+), 12 deletions(-) diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 31f2901a6a3..7c0ee87b9f4 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -25,6 +25,8 @@ The five-minute outage is a resilience test, not an HTTP downtime allowance. This suite gives its Daytona sandbox a 15-minute provider auto-stop interval, leaving headroom for the outage and replacement startup. Native agent idle timing, ownership/reconnect limits, archive/delete backstops and owned cleanup remain unchanged. +The retained continuation accepts whitespace between its exact marker and the +exact `queue_r` output token; readiness and final grading use the same matcher. The explicit-only [GitHub review bot journey](GITHUB-REVIEW-JOURNEY.md) tests attended bot setup and an automatic PR review against an attached deployment. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index f663eae75ba..64628298d7f 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -61,7 +61,7 @@ describe("runner E2E catalog", () => { } expect(baseConfig.autoStopInterval).toBe(5); expect(daytonaWarmEnvironment.buildEnvironment(environmentInput).config).toMatchObject({ autoStopInterval: 5 }); - expect(suite.definitionMetadata).toMatchObject({ version: 4, controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15 }); + expect(suite.definitionMetadata).toMatchObject({ version: 5, controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15 }); }); it.each([ diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 3b1306c9237..ebaf9b291af 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1173,7 +1173,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ })), tasks: everydayTasks.filter(task => task.id === "recover-controller") .map(task => ({ ...task, expectedRunCount: task.expectedRunCount + 1, automaticRetryPolicy: "single_attempt" as const })), expectedMatrixSize: 2, - definitionMetadata: { version: 4, scheduling: "explicit-only", controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, retainedControllerContinuation: true, + definitionMetadata: { version: 5, scheduling: "explicit-only", controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, retainedControllerContinuation: true, maximumAttemptsPerCell: 1, budgetMonthlyCents: 1000, instructions: "production", grading: "same-active-process-lease-native-prefix-and-workspace-receipt", authentication: "api-key", stagingAuthentication: "subscription", stagingQualification: false }, }, diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index 96ddde1277f..649b3217d8a 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -45,6 +45,8 @@ import { storyLifecycleChecks, storyRepliesConsumed, storyHasAgentReply, + storyAgentReplyMatches, + retainedControllerReplyMatcher, storyHasPendingHumanInteraction, storyReviewContinuationTimeoutDetail, storyHasStrandedBlockedLeaf, @@ -60,6 +62,7 @@ import { type StoryCheck, type StoryIssue, type StoryRun, + type StoryReplyExpectation, } from "./everyday-observations.js"; type Row = Record; @@ -325,7 +328,7 @@ export async function runEverydayFlow(input: Input) { sameConnections: false, }; } - async function settled(expectedAgentReply?: string) { + async function settled(expectedAgentReply?: StoryReplyExpectation) { const settledState = await pollUntil({ label: `everyday ${caseId} settled`, deadlineAt: input.deadlineAt, @@ -390,9 +393,7 @@ export async function runEverydayFlow(input: Input) { if (expectedAgentReply) { const issue = settledState.issues.find((candidate) => candidate.id === parent?.id); const reply = issue?.comments?.find( - (comment) => - comment.authorAgentId === fixtures.agent.id && - String(comment.body ?? "").includes(expectedAgentReply), + (comment) => storyAgentReplyMatches(comment, fixtures.agent.id, expectedAgentReply), ); if (reply) { // Keep the exact snapshot that satisfied the readiness predicate. A @@ -1455,15 +1456,16 @@ export async function runEverydayFlow(input: Input) { const outage = await input.restart(); await openParent(); const requestedAt = new Date().toISOString(), marker = `CONTINUED_${nonce}`; + const expectedReply = retainedControllerReplyMatcher(marker); await reply(`Run the existing delivered slugify.py on " Queue--Ready!! " with --separator _ and --max-length 7. Report its output after the marker ${marker}. Use the saved project; do not rebuild it.`); - await settled(`${marker} queue_r`); + await settled(expectedReply); const after = ev.runs.filter(run => !priorIds.has(run.id) && run.status === "succeeded" && run.agentId === fixtures.agent.id) .sort((a, b) => Date.parse(b.finishedAt ?? "") - Date.parse(a.finishedAt ?? ""))[0]; if (!after) throw new Error("Retained continuation produced no new successful run"); const events = await collectRunEvents((afterSeq, limit) => api.get(`/api/heartbeat-runs/${after.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); const evidence = { before, after: retainedControllerRun(after), outage: outage || undefined, requestedAt, completedTool: events.some(row => row.payload?.prpEvent?.eventType === "tool.execution.completed" && row.payload.prpEvent.payload?.status === "completed"), - usefulReply: settledAgentReply?.authorAgentId === fixtures.agent.id && String(settledAgentReply.body ?? "").includes(`${marker} queue_r`) }; + usefulReply: storyAgentReplyMatches(settledAgentReply, fixtures.agent.id, expectedReply) }; ev.checks.push(...gradeRetainedControllerContinuation(evidence)); await input.evidence("retained-controller-continuation.json", evidence); } diff --git a/tests/runner-e2e/everyday-observations.ts b/tests/runner-e2e/everyday-observations.ts index 40ab35e4a44..b3c78f8dabc 100644 --- a/tests/runner-e2e/everyday-observations.ts +++ b/tests/runner-e2e/everyday-observations.ts @@ -344,17 +344,34 @@ export function storyRepliesConsumed( ); } +export type StoryReplyExpectation = string | ((body: string) => boolean); + +export function storyAgentReplyMatches( + comment: StoryComment | undefined, + agentId: string, + expected: StoryReplyExpectation, +): boolean { + if (!comment || comment.authorAgentId !== agentId) return false; + const body = String(comment.body ?? ""); + return typeof expected === "string" ? body.includes(expected) : expected(body); +} + +/** The requested output is one exact token after this continuation's marker. */ +export function retainedControllerReplyMatcher(marker: string): (body: string) => boolean { + const escaped = marker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const pattern = new RegExp(`(?:^|\\s)${escaped}\\s+queue_r\\s*$`); + return body => pattern.test(body); +} + /** A terminal task is only ready for response assertions after its agent reply is visible. */ export function storyHasAgentReply( issue: StoryIssue | undefined, agentId: string, - expectedText: string, + expectedText: StoryReplyExpectation, ): boolean { return ( issue?.comments?.some( - (comment) => - comment.authorAgentId === agentId && - String(comment.body ?? "").includes(expectedText), + (comment) => storyAgentReplyMatches(comment, agentId, expectedText), ) ?? false ); } diff --git a/tests/runner-e2e/everyday.test.ts b/tests/runner-e2e/everyday.test.ts index 88cbecddd72..ebbc5acccc4 100644 --- a/tests/runner-e2e/everyday.test.ts +++ b/tests/runner-e2e/everyday.test.ts @@ -9,6 +9,8 @@ import { storyLifecycleChecks, storyRepliesConsumed, storyHasAgentReply, + storyAgentReplyMatches, + retainedControllerReplyMatcher, storyHasPendingAgentReview, storyHasPendingAgentWake, storyHasPendingAgentWork, @@ -385,6 +387,42 @@ describe("lifecycle oracle calibrated failures", () => { }); describe("reply completion boundary", () => { + it("accepts an explicit reply matcher for continuation output on a new line", () => { + const issue = { id: "parent", companyId: "company", title: "Continue", status: "done", + comments: [{ authorAgentId: "lead", body: "CONTINUED_nonce\nqueue_r" }] }; + const expected = (body: string) => body === "CONTINUED_nonce\nqueue_r"; + expect(storyHasAgentReply(issue, "lead", expected)).toBe(true); + }); + + it.each([" ", "\n", "\r\n", "\t", " \n"])("accepts continuation output separated by %j", separator => { + const expected = retainedControllerReplyMatcher("CONTINUED_nonce.1"); + const comment = { authorAgentId: "lead", body: `CONTINUED_nonce.1${separator}queue_r` }; + expect(storyHasAgentReply({ id: "task", companyId: "company", title: "Continue", status: "done", comments: [comment] }, "lead", expected)).toBe(true); + expect(storyAgentReplyMatches(comment, "lead", expected)).toBe(true); + }); + + it.each([ + "CONTINUED_nonce.1 queue_ready", "CONTINUED_nonce.1 queue_r_extra", "CONTINUED_nonce.1 wrong", + "CONTINUED_other queue_r", "CONTINUED_nonceX1 queue_r", "queue_r", "CONTINUED_nonce.1queue_r", + "OTHER_CONTINUED_nonce.1 queue_r", "CONTINUED_nonce.1 queue_r\nqueue_ready", + ])("rejects an incorrect continuation reply %j", body => { + const expected = retainedControllerReplyMatcher("CONTINUED_nonce.1"); + const comment = { authorAgentId: "lead", body }; + expect(storyHasAgentReply({ id: "task", companyId: "company", title: "Continue", status: "done", comments: [comment] }, "lead", expected)).toBe(false); + expect(storyAgentReplyMatches(comment, "lead", expected)).toBe(false); + }); + + it("preserves literal string expectations and continuation author checks", () => { + const expected = retainedControllerReplyMatcher("CONTINUED_nonce"); + const comment = { authorAgentId: "other", body: "CONTINUED_nonce\nqueue_r" }; + expect(storyAgentReplyMatches(comment, "lead", expected)).toBe(false); + const issue = { id: "task", companyId: "company", title: "Continue", status: "done", comments: [comment] }; + expect(storyHasAgentReply(issue, "lead", expected)).toBe(false); + expect(storyAgentReplyMatches({ ...comment, authorAgentId: "lead" }, "lead", "CONTINUED_nonce queue_r")).toBe(false); + expect(storyAgentReplyMatches({ authorAgentId: "lead", body: "a.b" }, "lead", "a.b")).toBe(true); + expect(storyAgentReplyMatches({ authorAgentId: "lead", body: "axb" }, "lead", "a.b")).toBe(false); + }); + const run: StoryRun = { id: "old", companyId: "company",