diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 9c9c1bf879b..f0432a659a1 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -168,6 +168,10 @@ lifecycle/performance events are insufficient. The startup case must stop after a process launch request but before a provider turn starts. Missing the boundary fails the case instead of silently testing another phase. +Restart continuity requires the agent to recall a phrase after the server +restarts. The final prompt does not reveal that phrase. A generic successful +reply after restart cannot pass this check. + The committed-send case withholds the browser's comment acknowledgement until the agent has saved one backlog task, drops that response, restarts Paperclip, and replays the exact public request with the original client request ID. It diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index e0335e0af80..b459d3a5af1 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -948,7 +948,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ profiles: runnerProfiles.filter(profile => ["legacy-codex", "legacy-claude", "runner-codex", "runner-acpx-claude"].includes(profile.id)).map(defaultPermissionProfile), environments: [localEnvironment], tasks: chatTasks, expectedMatrixSize: 28, excludedExecutionIds: ["legacy-codex", "legacy-claude"].flatMap(profile => ["reassign-task", "create-backlog"].map(task => `agent-chat.${profile}.local.${task}`)), - definitionMetadata: { version: 5, resetRunsCountedSeparately: true, permissions: "production-defaults", stopBoundary: "provider-turn-started" }, + definitionMetadata: { version: 6, resetRunsCountedSeparately: true, permissions: "production-defaults", stopBoundary: "provider-turn-started", restartMemory: "required-after-restart" }, }, { id: "agent-chat-hardening", label: "Agent Chat Recovery and Coordination", manualOnly: true, @@ -959,7 +959,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment, daytonaWarmEnvironment], tasks: chatHardeningTasks, expectedMatrixSize: 18, excludedExecutionIds: ["runner-codex", "runner-acpx-claude"].flatMap(profile => ["stop-startup-new-resume", "hire-delegate-reuse", "blocked-status-review"].map(task => `agent-chat-hardening.${profile}.daytona.${task}`)), - definitionMetadata: { version: 1, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only" }, + definitionMetadata: { version: 2, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart" }, }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), { diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index a8c3277c0cb..fd6f1a1f799 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -8,7 +8,7 @@ import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; import { isBlockedUnstartedWake } from "./non-execution-wake.js"; import { chatMarker } from "./chat-cases.js"; -import { assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js"; +import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js"; // Public API observations only: this driver never fabricates provider results or writes DB state. export interface ChatIssue { @@ -409,9 +409,14 @@ export async function runChatFlow(input: ChatFlowInput) { await idle(2); expect(runs).toHaveLength(count); await turn( - `We are done discussing it. Reply with ${marker} only; no further work.`, + `What phrase did I ask you to remember earlier? Reply with that remembered phrase followed by ${marker}; no further work.`, 3, ); + assertChatRememberedAfterRestart( + (await comments()).filter((comment) => comment.authorAgentId).at(-1)?.body ?? "", + secret, + marker, + ); } else { let cancelledId: string | undefined; if (stopCase) { diff --git a/tests/runner-e2e/chat-hardening.test.ts b/tests/runner-e2e/chat-hardening.test.ts index 7721870b3db..790fb9c4cb3 100644 --- a/tests/runner-e2e/chat-hardening.test.ts +++ b/tests/runner-e2e/chat-hardening.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { assertChatHire, assertChatSourceReview, assertChatStartupStopped, assertCommittedSendRetry, assertGroundedChatStatus, isChatStopReady } from "./chat-hardening.js"; +import { assertChatHire, assertChatRememberedAfterRestart, assertChatSourceReview, assertChatStartupStopped, assertCommittedSendRetry, assertGroundedChatStatus, isChatStopReady } from "./chat-hardening.js"; import { runnerMatrix } from "./catalog.js"; import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; @@ -10,6 +10,11 @@ const run = { id: "run", companyId: "company", agentId: "hire", status: "succeed const hiring = { agents: [{ id: "lead", name: "Lead", adapterConfig: { model: "model" } }, hire], leadId: "lead", hireName: "Morgan", hiredId: "hire", connectionId: "account", binding, taskIds: ["task"], tasks: [task], runs: [run] }; describe("agent chat hardening oracles", () => { + it("requires remembered context after restart, not just a new reply", () => { + expect(() => assertChatRememberedAfterRestart("OLDCONTEXT123 CHAT123", "OLDCONTEXT123", "CHAT123")).not.toThrow(); + expect(() => assertChatRememberedAfterRestart("CHAT123", "OLDCONTEXT123", "CHAT123")).toThrow(); + expect(() => assertChatRememberedAfterRestart("OLDCONTEXT999 CHAT123", "OLDCONTEXT123", "CHAT123")).toThrow(); + }); it("grades actual execution by the original hired identity and managed account", () => { expect(() => assertChatHire(hiring)).not.toThrow(); for (const wrong of [ diff --git a/tests/runner-e2e/chat-hardening.ts b/tests/runner-e2e/chat-hardening.ts index 09f5a379caf..21df0ede64f 100644 --- a/tests/runner-e2e/chat-hardening.ts +++ b/tests/runner-e2e/chat-hardening.ts @@ -76,6 +76,11 @@ export function isChatStopReady(events: Array<{ eventType?: unknown }>, phase: " return phase === "active" ? active : !active && events.some(event => event.eventType === "native.process_start_requested"); } +export function assertChatRememberedAfterRestart(reply: string, remembered: string, marker: string) { + expect(reply).toContain(remembered); + expect(reply).toContain(marker); +} + export function assertChatStartupStopped(run: ChatRun, events: Array<{ eventType?: unknown; createdAt?: string }>) { const cancellation = run.resultJson?.nativeCancellation as Record | undefined; const started = events.filter(event => ["turn.started", "turn.accepted"].includes(String(event.eventType)));