diff --git a/doc/evals.md b/doc/evals.md index 20f89339176..e2db0f49f9e 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -117,8 +117,10 @@ successor. Registration and calibration are separate from a live staging pass. deployment targets and starting-state recipes. The shared task-properties journey and task-document create/edit/reopen journey use no model; Cloud onboarding supplies reusable verified state. The document journey checks exact -saved content and both revision records through independent API reads. Cloud -fleet operations and trusted CI remain private. Local eval gateway qualification +saved content and both revision records through independent API reads. The explicit `chat-continuity` selection verifies browser reload/reopen, prompt-hidden recall, +ordinary `/new` reset, fresh native context and retained history against an attached target. +It requires a fresh owned agent with no prior work runs and uses three provider turns plus one synthetic reset row; it does not restart a shared stack. +Cloud fleet operations and trusted CI remain private. Local eval gateway qualification does not qualify Cloud routes, and these selections stay outside `--all`. The explicit-only [case handoff workflow](../tests/runner-e2e/README.md#persistent-case-handoffs-explicit-only) diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 80c30828b23..6fb291d0ac7 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -31,6 +31,7 @@ never start a server or depend on a tenant filesystem. | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | | `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | +| `chat-continuity` | Fresh owned native agent with no prior work runs; Agent Chat enabled; fresh owned user chat; three provider turns and one synthetic reset | Recall prompt-hidden context after reopening, then use `/new` to advance the same saved conversation to a fresh native session, preserve history and verify the final reply after reload. See [CHAT-CONTINUITY.md](product-journeys/CHAT-CONTINUITY.md) | | `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | | `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | | `task-edit-lifecycle` | Company, signed-in board user, task-write capability; target support for run-owned labels and exact predeclared `taskEdit.revisedTitle` cleanup; zero model runs | Create a self-assigned task, edit its inline header title and label, reload, complete and reopen it; independently verify exact saved state, unchanged context and zero runs. Open a closed Properties panel through its shipped toggle | diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 93833491ec8..4ab07867b7f 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -136,6 +136,7 @@ describe("runner E2E catalog", () => { "task-create", "project-task-context", "ask-question", + "chat-continuity", "question-answer-resume", "task-properties", "task-document-persistence", diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index ab33d9780cd..b964103c45d 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -26,6 +26,7 @@ associate(["product-journeys"], { "inbox-triage": ["inbox-search/views", "inbox-search/triage"], "task-create": ["tasks/create"], "ask-question": ["questions-and-approvals/questions"], "question-answer-resume": ["questions-and-approvals/questions", "questions-and-approvals/continuation"], "followup-while-running": ["steering/queue"], + "chat-continuity": ["agent-chat/history", "runs-adapters/session"], "hire-delegate-reuse": ["agents/hire", "documents-artifacts/documents"], "agent-settings-persistence": ["agents/instructions", "tasks/completion"], "project-task-context": ["projects/lifecycle", "projects/task-context", "tasks/create"], diff --git a/tests/runner-e2e/product-journeys/CHAT-CONTINUITY.md b/tests/runner-e2e/product-journeys/CHAT-CONTINUITY.md new file mode 100644 index 00000000000..003c2fd19e2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/CHAT-CONTINUITY.md @@ -0,0 +1,62 @@ +# Chat reload and session continuity + +`chat-continuity` is a deployment-targeted Product E2E journey. It uses the +ordinary Agent Chat composer on a fresh, campaign-owned native agent. It does +not restart a persistent Cloud instance or alter a shared source agent. +The agent-filtered heartbeat history must be empty before the first send. +An agent with earlier completed, failed, queued or active work is unsupported; +the fixture does not subtract or hide those runs. A target that reuses one +agent across several journeys must supply a separate fresh agent for this case. + +The first exchange contains a random phrase. After leaving the chat, reopening +it and reloading, the next message asks for that phrase without supplying it. +The independently attributed reply must contain the phrase, and both native +turns must retain the same session identity. + +The user then sends the ordinary `/new` command. Paperclip preserves the same +backing chat and saved history while advancing `conversationSessionGeneration` +from zero to one. The final request does not reveal the remembered phrase. Its +reply must use the fresh-context marker, with a different native session, no +prior session on admission, and no old phrase in the new native task context. +The independent run detail must include its `sessionIdBefore` column explicitly +set to `null`; a missing or empty field is not session-boundary evidence. +Saved history is not removed or rewritten by the fixture. + +Every reply is read from the actual rendered chat after reload using the +existing run-bound native reply oracle. Board-user messages must each persist +exactly once and bind to exactly one observed run. The original comments remain +unchanged through the final reload. The owned agent must create no extra task, +project or teammate. + +The closed accounting contract is **three successful native provider turns +plus one synthetic reset row**, four persisted run IDs in total. The reset must +have its normal reset context and a matching original `/issues/:id/runs` +summary receipt and must not publish an assistant response. That receipt binds +the request's company and issue to the same successful summary run, agent, +context issue and wake comment as the independently read detail. The raw run +detail is not rewritten. Its optional `resultJson` can be absent or compact; +an explicit contradictory reset flag is rejected. Real provider turns require +the authoritative committed `nativePhase`, not an optional duplicate result +field. Their output still requires a bound native-final event or the selected +attributed completion comment and an actual reloaded rendered reply. +The reset is not counted as an additional provider turn. Source model, provider and +managed environment must match all three native turns. The target provider owns +authentication, the disposable agent, pre-send chat ownership, complete run and +usage collection, stopping and evidence-first cleanup. Unknown setup or model +cost remains unknown; a run limit is not proof of an absolute spend limit. + +The company activity request includes the exact owned agent filter before the +server's 500-row limit. Every returned row must match that company and agent, +have its canonical activity identity, and leave the response below that limit. +Mixed or truncated results fail; the client never filters a global response +into an apparent pass. Unrelated company history therefore does not limit this +fresh agent's proof, while any owned task, project or teammate creation fails. +Activity identifies the acting agent, not the created entity. Board provisioning +therefore has no agent actor even when its `entityId` is this new agent. Creation +attributed to the owned agent is rejected, including activity before the first +chat message; the fresh-agent contract does not grandfather that work. + +This case covers browser reload/reopen and explicit session reset. Controller +restart remains covered by the separate existing Agent Chat journeys. Local +oracle and catalog checks do not establish a staging pass; daily admission +requires the actual original staging attempt and complete cleanup evidence. diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index b73a4cda8d5..5fd94765f42 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -24,6 +24,7 @@ type AgentJourney = Exclude; +const allPass = (f: ReturnType) => gradeChatContinuity(f.scope, f.points).every(check => check.passed); + +describe("chat reload and new-session evidence", () => { + it("requires the actual three-turn outcome, one reset and run-bound reloaded replies", () => { + const f = chatContinuityFixture(); + expect(gradeChatContinuity(f.scope, f.points).map(row => row.id)).toEqual(CHAT_CONTINUITY_CHECKS); + expect(allPass(f)).toBe(true); + }); + it("accepts normal compact run details only with bound native-final events and the original reset summary", () => { + expect(allPass(compactChatContinuityFixture())).toBe(true); + }); + it("rejects an unscoped activity response even below the truncation limit", () => { + const f = chatContinuityFixture(); + f.points[5]!.activity.push({ id: "foreign-event", companyId: "foreign", agentId: "other-agent", action: "project.created", + entityType: "project", entityId: "foreign-project", runId: "other-run" }); + expect(allPass(f)).toBe(false); + }); + it("requires the fresh agent's pre-send empty-history receipt", () => { + const f = chatContinuityFixture(); delete f.scope.agentHistory; + expect(allPass(f)).toBe(false); + }); + it("never supplies the old phrase in the recall or fresh-context request", () => { + const s = chatContinuityScenario("random-123"); + expect(s.remember).toContain(s.remembered); + expect(s.recall).not.toContain(s.remembered); + expect(s.fresh).not.toContain(s.remembered); + expect(s.reset).toBe("/new"); + }); + it("registers the exact measured checks and counts all four persisted runs", () => { + expect(journeyRequiredChecks["chat-continuity"]).toEqual(CHAT_CONTINUITY_CHECKS); + const config = { version: 1, id: "qa-chat", mode: "existing-cloud", adapterModule: "/private/provider.mjs", + executionEnvironment: "daytona", journeys: ["chat-continuity"], baseline: { features: {}, connections: [] }, + limits: { durationMs: 900_000, maxRuns: 4, modelBudgetCents: 500 } }; + expect(parseTargetConfig(config).limits.maxRuns).toBe(4); + for (const maxRuns of [0, 3, 5]) expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns } })).toThrow("agent_journeys_require_bounded_runs"); + }); + const mutants: ReadonlyArray) => void]> = [ + ["missing point", (f: ReturnType) => { f.points.pop(); }], + ["duplicate phase", (f) => { f.points[5]!.phase = "fresh"; }], + ["foreign chat", (f) => { f.points[1]!.chat.companyId = "foreign"; }], + ["different user", (f) => { f.points[3]!.chat.conversationUserId = "foreign"; }], + ["replaced backing issue", (f) => { f.points[5]!.chat.id = "replacement"; }], + ["active conversation", (f) => { f.points[5]!.chat.conversationState = "running"; }], + ["scheduled retry", (f) => { f.points[5]!.chat.scheduledRetry = { state: "pending" }; }], + ["duplicate user send", (f) => { f.points[5]!.comments.push({ ...f.points[5]!.comments[0], id: "duplicate" }); }], + ["foreign author", (f) => { f.points[5]!.comments[0]!.authorUserId = "foreign"; }], + ["wrong remembered phrase", (f) => { f.points[2]!.reply!.body = "I do not remember"; }], + ["unbound first reply", (f) => { f.points[0]!.reply!.anchor = "unbound"; }], + ["first reply missing after reopen", (f) => { delete f.points[1]!.reply; }], + ["stale recall response", (f) => { f.points[2]!.reply!.runId = "run-0"; }], + ["foreign recall run", (f) => { f.points[2]!.runs[1]!.agentId = "foreign"; }], + ["different provider", (f) => { f.points[2]!.runs[1]!.runnerProfileJson.nativeExecutionInput.provider.kind = "other"; }], + ["different model", (f) => { f.points[5]!.runs[3]!.runnerProfileJson.nativeExecutionInput.provider.model = "other"; }], + ["different environment", (f) => { f.points[5]!.runs[3]!.contextSnapshot.paperclipEnvironment.id = "other"; }], + ["uncommitted native phase", (f) => { f.points[5]!.runs[3]!.nativePhase = "finalizing"; }], + ["missing session proof", (f) => { delete f.points[0]!.runs[0]!.nativeSessionId; }], + ["unintended session replacement on reload", (f) => { f.points[2]!.runs[1]!.nativeSessionId = "replacement"; }], + ["reset generation absent", (f) => { f.points[3]!.chat.conversationSessionGeneration = 0; }], + ["fresh run generation stale", (f) => { f.points[4]!.runs[3]!.contextSnapshot.conversationSessionGeneration = 0; }], + ["fresh turn resumes old session", (f) => { f.points[4]!.runs[3]!.nativeSessionId = "original-session"; }], + ["previous session supplied on fresh admission", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = "original-session"; }], + ["missing fresh-admission session evidence", (f) => { delete f.points[4]!.runs[3]!.sessionIdBefore; }], + ["undefined fresh-admission session evidence", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = undefined; }], + ["empty fresh-admission session evidence", (f) => { f.points[4]!.runs[3]!.sessionIdBefore = ""; }], + ["old phrase leaked in native context", (f) => { f.points[4]!.runs[3]!.contextSnapshot.paperclipTaskMarkdown += f.scenario.remembered; }], + ["old phrase returned after reset", (f) => { f.points[4]!.reply!.body = f.scenario.remembered; }], + ["fresh marker absent", (f) => { f.points[4]!.reply!.body = "I have no phrase"; }], + ["extra run", (f) => { f.points[5]!.runs.push({ ...f.points[5]!.runs[3], id: "extra" }); }], + ["duplicate run ID", (f) => { f.points[5]!.runs[3]!.id = "run-0"; }], + ["missing reset", (f) => { f.points[5]!.runs.splice(2, 1); }], + ["missing synthetic reset receipt", (f) => { f.points[5]!.resetReceipts.splice(0); }], + ["foreign reset summary agent", (f) => { f.points[5]!.resetReceipts[0]!.agentId = "foreign"; }], + ["foreign reset summary company", (f) => { f.points[5]!.resetReceipts[0]!.companyId = "foreign"; }], + ["foreign reset summary issue", (f) => { f.points[5]!.resetReceipts[0]!.contextIssueId = "foreign"; }], + ["wrong reset summary run", (f) => { f.points[5]!.resetReceipts[0]!.runId = "run-0"; }], + ["wrong reset summary wake", (f) => { f.points[5]!.resetReceipts[0]!.wakeCommentId = "user-0"; }], + ["reset summary not successful", (f) => { f.points[5]!.resetReceipts[0]!.status = "running"; }], + ["contradictory reset detail", (f) => { f.points[5]!.runs[2]!.resultJson.conversationReset = false; }], + ["provider work disguised as reset", (f) => { f.points[5]!.runs[2]!.runnerProfileJson = { nativeExecutionInput: { provider: { kind: f.scope.provider, model: f.scope.model } } }; }], + ["native session attached to synthetic reset", (f) => { f.points[5]!.runs[2]!.nativeSessionId = "unexpected-provider-session"; }], + ["reset publishes an answer", (f) => { f.points[5]!.comments.push({ id: "reset-answer", authorAgentId: f.scope.agentId, createdByRunId: "run-2", body: "Hi" }); }], + ["unrequested completion callback", (f) => { f.points[5]!.runs[3]!.contextSnapshot.source = "automation"; }], + ["wrong wake comment", (f) => { f.points[5]!.runs[3]!.contextSnapshot.wakeCommentId = "user-0"; }], + ["run has conflicting issue IDs", (f) => { f.points[5]!.runs[3]!.issueId = "foreign"; }], + ["changed original history", (f) => { f.points[5]!.comments[0]!.body = "changed"; }], + ["missing original history", (f) => { f.points[5]!.comments.splice(1, 1); }], + ["reopen repeats the send", (f) => { f.points[5]!.runs.push({ ...f.points[5]!.runs[3], id: "repeated" }); }], + ["reply not observed after reload", (f) => { f.points[5]!.reply!.observedAfterReload = false; }], + ["hidden final reply", (f) => { f.points[5]!.reply!.visible = false; }], + ["created work", (f) => { f.points[5]!.tasks.push({ id: "unexpected" }); }], + ["created project", (f) => { f.points[5]!.activity.push({ id: "created-project", companyId: f.scope.companyId, agentId: f.scope.agentId, + runId: "run-1", action: "project.created", entityType: "project", entityId: "unexpected" }); }], + ["truncated creation history", (f) => { f.points[5]!.activity = Array.from({ length: 500 }, () => ({})); }], + ["missing run collection", (f) => { f.points[5]!.runs = undefined as unknown as Row[]; }], + ["malformed history row", (f) => { f.points[5]!.comments.push(null as unknown as Row); }], + ]; + it.each(mutants)("rejects %s", (_name, mutate) => { + const f = chatContinuityFixture(); + mutate(f); + expect(allPass(f)).toBe(false); + }); + it("rejects another agent's activity rather than filtering an unscoped response", () => { + const f = chatContinuityFixture(); + f.points[5]!.activity.push({ companyId: f.scope.companyId, agentId: "other-agent", runId: "other-run", action: "project.created" }); + expect(allPass(f)).toBe(false); + }); + it("accepts complete owned activity and rejects unaccounted creation by that same agent", () => { + const f = chatContinuityFixture(); + const row = { id: "owned-event", companyId: f.scope.companyId, agentId: f.scope.agentId, + runId: "run-0", action: "heartbeat.run_completed", entityType: "heartbeat_run", entityId: "run-0" }; + f.points[5]!.activity.push(row); + expect(allPass(f)).toBe(true); + row.action = "project.created"; row.runId = "unaccounted-owned-run"; + expect(allPass(f)).toBe(false); + }); + it("does not treat a compact run as sufficient without actual native output proof", () => { + const f = compactChatContinuityFixture(); + f.points[5]!.events = []; + expect(allPass(f)).toBe(false); + }); + it("retains backward-compatible issue aliases only when they agree", () => { + expect(chatContinuityIssueId({ nativeIssueId: "chat", contextSnapshot: { issueId: "chat" } })).toBe("chat"); + expect(chatContinuityIssueId({ issueId: "foreign", nativeIssueId: "chat" })).toBeUndefined(); + expect(chatContinuityIssueId({})).toBeUndefined(); + }); +}); diff --git a/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts b/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts new file mode 100644 index 00000000000..598d76f3ff5 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-evidence.ts @@ -0,0 +1,157 @@ +import { chatMarker } from "../chat-cases.js"; +import { taskReplyBelongsToRun, type TaskReplyObservation } from "../native-task-reply.js"; + +type Row = Record; +export const CHAT_CONTINUITY_CHECKS = [ + "chat-fresh-owned-conversation", "chat-user-messages-stored-once", "chat-reload-preserved-context", + "chat-new-session-boundary", "chat-fresh-context-answer", "chat-history-preserved-after-reload", + "chat-no-unrequested-work", "chat-exact-turn-accounting", "configured-model-and-environment", +] as const; + +export function chatContinuityScenario(campaignId: string) { + const remembered = chatMarker("OLDCONTEXT", campaignId), marker = chatMarker("CHAT", campaignId); + return { + remembered, marker, + remember: `For this conversation only, remember the phrase ${remembered}. Just acknowledge briefly; no project or task is needed.`, + recall: "What phrase did I just ask you to remember? Reply with the phrase only. No project or task is needed.", + reset: "/new", + fresh: `Without reading older history or files, if you have a remembered phrase in your current context return it; otherwise reply exactly ${marker}. Do not look it up. No project or task is needed.`, + }; +} + +export interface ChatContinuityScope { + companyId: string; userId: string; agentId: string; agentName: string; + environmentId: string; provider: string; model: string; campaignId: string; + agentHistory?: { source: "company-agent-runs"; companyId: string; agentId: string; limit: 1; runIds: string[]; observedBeforeSend: true }; +} +export interface ChatContinuityResetReceipt { + source: "issue-runs"; companyId: string; issueId: string; + runId: string; agentId: string; status: string; contextIssueId: string; wakeCommentId: string; + resultJson: { conversationReset: true }; +} +export interface ChatContinuityPoint { + phase: "initial" | "returned" | "recalled" | "reset" | "fresh" | "reopened"; + chat: Row; runs: Row[]; comments: Row[]; tasks: Row[]; activity: Row[]; + reply?: TaskReplyObservation & { runId: string; agentId: string; observedAfterReload: boolean }; + events?: Row[]; + resetReceipts: ChatContinuityResetReceipt[]; +} +const nonempty = (value: unknown): value is string => typeof value === "string" && value.length > 0; +export const chatContinuityResetRun = (run: Row) => run.contextSnapshot?.conversationReset === true || run.resultJson?.conversationReset === true; +export const chatContinuityIssueId = (run: Row): string | undefined => { + const ids = [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].filter(nonempty); + return ids.length && new Set(ids).size === 1 ? ids[0] : undefined; +}; +const userBody = (row: Row) => typeof row.body === "string" ? row.body.replaceAll("\\_", "_").trim() : ""; +const sameIds = (left: Row[], right: Row[]) => left.length === right.length && left.every(row => right.some(other => row.id === other.id)); +const commentContent = (row: Row) => [row.id, row.companyId, row.issueId, row.body, row.authorAgentId, + row.authorUserId, row.createdByRunId, row.conversationSessionGeneration]; +const sameRows = (left: Row[], right: Row[]) => left.every(row => right.some(other => other.id === row.id + && JSON.stringify(commentContent(other)) === JSON.stringify(commentContent(row)))); + +export function chatContinuityActivityComplete(scope: Pick, rows: Row[]): boolean { + return Array.isArray(rows) && rows.length < 500 && new Set(rows.map(row => row?.id)).size === rows.length + && rows.every(row => row && nonempty(row.id) && row.companyId === scope.companyId && row.agentId === scope.agentId + && nonempty(row.action) && nonempty(row.entityType) && nonempty(row.entityId)); +} + +export function chatContinuityResetReceiptMatches(scope: Pick, + chatId: string, run: Row, receipt: ChatContinuityResetReceipt): boolean { + const fullResult = run.resultJson; + const optionalFlagValid = fullResult == null || typeof fullResult === "object" && !Array.isArray(fullResult) + && (!Object.hasOwn(fullResult, "conversationReset") || fullResult.conversationReset === true); + return receipt?.source === "issue-runs" && receipt.companyId === scope.companyId && receipt.issueId === chatId + && receipt.runId === run.id && receipt.agentId === scope.agentId && receipt.status === "succeeded" + && receipt.contextIssueId === chatId && nonempty(receipt.wakeCommentId) + && receipt.wakeCommentId === run.contextSnapshot?.wakeCommentId && receipt.resultJson?.conversationReset === true + && run.companyId === scope.companyId && run.agentId === scope.agentId && chatContinuityIssueId(run) === chatId + && run.status === "succeeded" && run.contextSnapshot?.conversationReset === true && optionalFlagValid + && !run.runnerProfileJson?.nativeExecutionInput && !run.nativeSessionId; +} + +export function gradeChatContinuity(scope: ChatContinuityScope, points: ChatContinuityPoint[]) { + const scenario = chatContinuityScenario(scope.campaignId); + const phases = ["initial", "returned", "recalled", "reset", "fresh", "reopened"] as const; + const complete = points.length === phases.length && points.every((point, index) => point?.phase === phases[index] + && point.chat && [point.runs, point.comments, point.tasks, point.activity, point.resetReceipts].every(rows => Array.isArray(rows) + && rows.every(row => row && typeof row === "object" && !Array.isArray(row)))); + if (!complete) return CHAT_CONTINUITY_CHECKS.map(id => ({ id, passed: false })); + const [initial, returned, recalled, reset, fresh, reopened] = complete ? points : []; + const chatId = initial?.chat?.id; + const scoped = complete && [scope.companyId, scope.userId, scope.agentId, scope.agentName, scope.environmentId, + scope.provider, scope.model, scope.campaignId].every(nonempty) && nonempty(chatId) + && scope.agentHistory?.source === "company-agent-runs" && scope.agentHistory.companyId === scope.companyId + && scope.agentHistory.agentId === scope.agentId && scope.agentHistory.limit === 1 && scope.agentHistory.observedBeforeSend === true + && Array.isArray(scope.agentHistory.runIds) && scope.agentHistory.runIds.length === 0 + && points.every(point => point.chat?.id === chatId && point.chat.companyId === scope.companyId + && point.chat.assigneeAgentId === scope.agentId && point.chat.conversationAgentId === scope.agentId + && point.chat.conversationUserId === scope.userId && point.chat.createdByUserId === scope.userId + && point.chat.title === `Chat with ${scope.agentName}` && point.chat.status === "in_review" + && point.chat.conversationState === "waiting" && !point.chat.executionRunId && !point.chat.scheduledRetry && !point.chat.activeRecoveryAction); + const requested = [scenario.remember, scenario.recall, scenario.reset, scenario.fresh]; + const finalComments = reopened?.comments ?? []; + const board = finalComments.filter(row => row.authorUserId === scope.userId && !row.authorAgentId); + const userMessages = scoped && board.length === requested.length && new Set(board.map(row => row.id)).size === requested.length + && requested.every(body => board.filter(row => userBody(row) === body && row.companyId === scope.companyId && row.issueId === chatId).length === 1) + && finalComments.every(row => !row.authorUserId || row.authorUserId === scope.userId); + const runs = reopened?.runs ?? [], providerRuns = runs.filter(run => !chatContinuityResetRun(run)), resetRuns = runs.filter(chatContinuityResetRun); + const runBinding = scoped && points.every(point => point.runs.every(run => nonempty(run.id) && run.companyId === scope.companyId + && run.agentId === scope.agentId && chatContinuityIssueId(run) === chatId && run.status === "succeeded")); + const providerBinding = runBinding && providerRuns.length === 3 && points.flatMap(point => point.runs) + .filter(run => !chatContinuityResetRun(run)).every(run => { + const provider = run.runnerProfileJson?.nativeExecutionInput?.provider; + return run.runtimeMode === "native" && provider?.kind === scope.provider && provider?.model === scope.model + && run.contextSnapshot?.paperclipEnvironment?.id === scope.environmentId + && run.nativePhase === "committed" + && nonempty(run.nativeSessionId); + }); + const expectedCounts = [1, 1, 2, 3, 4, 4]; + const exactTurns = runBinding && providerBinding && resetRuns.length === 1 && runs.length === 4 && new Set(runs.map(run => run.id)).size === 4 + && points.every(point => point.resetReceipts.length === point.runs.filter(chatContinuityResetRun).length + && point.runs.filter(chatContinuityResetRun).every(run => point.resetReceipts + .filter(receipt => chatContinuityResetReceiptMatches(scope, chatId, run, receipt)).length === 1)) + && points.every((point, index) => point.runs.length === expectedCounts[index] + && point.runs.every(run => runs.some(final => final.id === run.id && final.status === run.status))) + && runs.every(run => { + const request = board.find(comment => comment.id === run.contextSnapshot?.wakeCommentId); + return request && run.contextSnapshot?.source === "issue.comment" + && (chatContinuityResetRun(run) ? userBody(request) === scenario.reset : userBody(request) !== scenario.reset); + }) && new Set(runs.map(run => run.contextSnapshot?.wakeCommentId)).size === 4; + const pointReply = (point: ChatContinuityPoint | undefined) => { + const receipt = point?.reply, run = point?.runs.find(run => run.id === receipt?.runId); + return receipt && run && receipt.agentId === scope.agentId && receipt.observedAfterReload === true + && taskReplyBelongsToRun(receipt, run, scope.agentId, point!.comments, point!.events ?? []) ? receipt : undefined; + }; + const originalReply = pointReply(initial), returnedReply = pointReply(returned), recalledReply = pointReply(recalled), freshReply = pointReply(fresh), reopenedReply = pointReply(reopened); + const firstRuns = initial?.runs.filter(run => !chatContinuityResetRun(run)) ?? []; + const recallRuns = recalled?.runs.filter(run => !chatContinuityResetRun(run) && run.id !== firstRuns[0]?.id) ?? []; + const freshRuns = fresh?.runs.filter(run => !chatContinuityResetRun(run) && !recalled?.runs.some(old => old.id === run.id)) ?? []; + const first = firstRuns[0], second = recallRuns[0], last = freshRuns[0], resetRun = resetRuns[0]; + const recall = exactTurns && Boolean(originalReply && returnedReply && recalledReply && originalReply.runId === first?.id + && returnedReply.runId === originalReply.runId && returnedReply.body === originalReply.body && recalledReply.runId === second?.id + && recalledReply.body.includes(scenario.remembered) && first?.nativeSessionId === second?.nativeSessionId + && initial!.chat.conversationSessionGeneration === 0 && returned!.chat.conversationSessionGeneration === 0 + && recalled!.chat.conversationSessionGeneration === 0 && first?.contextSnapshot?.conversationSessionGeneration === 0 + && second?.contextSnapshot?.conversationSessionGeneration === 0 && sameIds(initial!.runs, returned!.runs) + && sameRows(initial!.comments, returned!.comments)); + const boundary = exactTurns && Boolean(first && second && last && resetRun && freshRuns.length === 1 + && reset!.chat.conversationSessionGeneration === 1 && fresh!.chat.conversationSessionGeneration === 1 + && reopened!.chat.conversationSessionGeneration === 1 && last.contextSnapshot?.conversationSessionGeneration === 1 + && resetRun.contextSnapshot?.conversationSessionGeneration === 1 + && Object.hasOwn(last, "sessionIdBefore") && last.sessionIdBefore === null + && last.nativeSessionId !== first.nativeSessionId && last.nativeSessionId !== second.nativeSessionId + && typeof last.contextSnapshot?.paperclipTaskMarkdown === "string" + && !last.contextSnapshot.paperclipTaskMarkdown.includes(scenario.remembered) + && finalComments.every(row => row.createdByRunId !== resetRun.id)); + const freshAnswer = boundary && Boolean(freshReply && freshReply.runId === last?.id + && freshReply.body.trim() === scenario.marker && !freshReply.body.includes(scenario.remembered)); + const history = scoped && Boolean(reopenedReply && freshReply && reopenedReply.runId === freshReply.runId + && reopenedReply.body === freshReply.body && sameIds(fresh!.runs, reopened!.runs) + && sameRows(initial!.comments, reopened!.comments) && sameRows(recalled!.comments, reopened!.comments) + && sameRows(fresh!.comments, reopened!.comments)); + const noWork = scoped && points.every(point => Array.isArray(point.tasks) && point.tasks.length === 0 + && chatContinuityActivityComplete(scope, point.activity) + && !point.activity.some(row => ["issue.created", "issue.child_created", "project.created", "agent.created", "agent.hire_created"].includes(row.action))); + const values = [scoped, userMessages, recall, boundary, freshAnswer, history, noWork, exactTurns, providerBinding]; + return CHAT_CONTINUITY_CHECKS.map((id, index) => ({ id, passed: Boolean(values[index]) })); +} diff --git a/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts b/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts new file mode 100644 index 00000000000..f69c7f69bcf --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-flow.test.ts @@ -0,0 +1,180 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; +import { chatContinuityFixture, compactChatContinuityFixture } from "./chat-continuity.test-fixture.js"; + +const mocks = vi.hoisted(() => ({ send: vi.fn(), reply: vi.fn(), capture: vi.fn() })); +vi.mock("../chat-flow.js", () => ({ sendChatMessage: mocks.send })); +vi.mock("../native-task-reply.js", async importOriginal => ({ + ...await importOriginal(), readBoundTaskFinalReply: mocks.reply, +})); +vi.mock("./product-screenshot.js", () => ({ captureProductScreenshot: mocks.capture })); +vi.mock("@playwright/test", () => ({ expect: (locator: { isVisible(): Promise }) => ({ + async toBeVisible() { expect(await locator.isVisible()).toBe(true); }, +}) })); +import { runChatContinuity } from "./chat-continuity-flow.js"; + +const directories: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + vi.resetAllMocks(); + for (const directory of directories.splice(0)) await rm(directory, { recursive: true, force: true }); +}); +async function fixture(options: { preexisting?: boolean; intentFails?: boolean; sendFails?: boolean; foreignRun?: boolean; + compactRuns?: boolean; unrelatedActivity?: number; ignoreActivityScope?: boolean; priorAgentStatus?: string; mixedAgentHistory?: boolean; + provisioningActor?: "board" | "owned-agent" } = {}) { + const f = options.compactRuns ? compactChatContinuityFixture() : chatContinuityFixture(); + const directory = await mkdtemp(path.join(os.tmpdir(), "chat-continuity-flow-")); + directories.push(directory); + let stage = options.preexisting ? 0 : -1; + const messages: string[] = [], actions: string[] = []; + const realTimeout = globalThis.setTimeout; + vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => + realTimeout(callback, delay === 1000 ? 0 : delay, ...args)) as typeof setTimeout); + const current = () => f.points[stage]!; + const companyActivity = [ + ...Array.from({ length: options.unrelatedActivity ?? 0 }, (_, index) => ({ id: `unrelated-${index}`, + companyId: f.scope.companyId, agentId: "other-agent", runId: "other-run", action: "project.created", entityType: "project", entityId: `other-project-${index}` })), + ...(options.provisioningActor ? [{ id: "provisioned-agent", companyId: f.scope.companyId, + agentId: options.provisioningActor === "board" ? null : f.scope.agentId, runId: null, + action: "agent.created", entityType: "agent", entityId: f.scope.agentId }] : []), + ]; + const companyRuns = Array.from({ length: options.unrelatedActivity ?? 0 }, (_, index) => ({ id: `unrelated-run-${index}`, + companyId: f.scope.companyId, agentId: "other-agent", status: "succeeded" })); + if (options.priorAgentStatus) companyRuns.push({ id: "prior-owned-run", companyId: f.scope.companyId, agentId: f.scope.agentId, status: options.priorAgentStatus }); + const get = vi.fn(async (route: string) => { + if (route === "/api/instance/settings/experimental") return { enableAgentChat: true }; + if (route === "/api/companies/company/heartbeat-runs?agentId=agent&limit=1&summary=true") { + actions.push("history"); + if (options.mixedAgentHistory) return [{ id: "foreign-run", companyId: "foreign-company", agentId: "other-agent", status: "running" }]; + return companyRuns.filter(run => run.agentId === f.scope.agentId).slice(0, 1); + } + if (route === "/api/companies/company/chats/agent") return stage < 0 ? null : structuredClone(current().chat); + if (route === "/api/issues/chat/runs") return current().runs.map(run => ({ runId: run.id, agentId: run.agentId, status: run.status, + contextIssueId: run.contextSnapshot.issueId, wakeCommentId: run.contextSnapshot.wakeCommentId, + resultJson: run.contextSnapshot.conversationReset ? { conversationReset: true } : {} })); + if (route === "/api/issues/chat/comments?order=asc") return structuredClone(current().comments); + if (route === "/api/companies/company/issues?view=index&assigneeAgentId=agent&includeAgentCreatedTasks=true&limit=2") return []; + if (route === "/api/companies/company/activity?agentId=agent&limit=500") { + return (options.ignoreActivityScope ? companyActivity : companyActivity.filter(row => row.agentId === f.scope.agentId)).slice(0, 500); + } + if (/\/api\/heartbeat-runs\/run-\d\/events\?/.test(route)) return structuredClone(current().events ?? []); + const id = route.match(/^\/api\/heartbeat-runs\/(run-\d)$/)?.[1]; + if (id) { + const run = structuredClone(current().runs.find(row => row.id === id)!); + if (options.foreignRun) run.nativeIssueId = "foreign"; + return run; + } + throw new Error(`unexpected GET ${route}`); + }); + const page = { goto: vi.fn(async (_url: string) => { actions.push("navigate"); }), reload: vi.fn(async () => { actions.push("reload"); }), + getByTestId: vi.fn(() => ({ isVisible: async () => true })) }; + const own = vi.fn(async (resource: { id?: string }) => { + actions.push(resource.id ? "receipt" : "intent"); + if (options.intentFails && !resource.id) throw new Error("ownership intent failed"); + }); + const context = { page, api: { get, origin: "https://qa.example.test" }, + company: { id: f.scope.companyId, issuePrefix: "QA" }, user: { id: f.scope.userId, name: "QA" }, + environmentId: f.scope.environmentId, agent: { id: f.scope.agentId, name: f.scope.agentName, + provider: f.scope.provider, model: f.scope.model, runtimeMode: "native" } } as unknown as JourneyContext; + const evidence: JourneyEvidence = { outcome: "passed", startingState: "recipe", phases: [], checks: [], resourceDisposition: "released", privacy: "private" }; + mocks.send.mockImplementation(async (_page, message: string) => { + actions.push("send"); messages.push(message); + if (options.sendFails) throw new Error("uncertain send response"); + const index = [f.scenario.remember, f.scenario.recall, f.scenario.reset, f.scenario.fresh].indexOf(message); + expect(index).toBe(messages.length - 1); + stage = [0, 2, 3, 4][index]!; + }); + mocks.reply.mockImplementation(async input => { + await input.page.reload(); + const data = await input.load(); + expect(data.run.id).toBe(input.runId); + const point = f.points.find(point => point.reply?.runId === input.runId)!; + return structuredClone(point.reply!); + }); + mocks.capture.mockResolvedValue(undefined); + return { ...f, context, evidence, own, directory, messages, actions, page, get, companyActivity, + run: () => runChatContinuity(context, { own } as unknown as TargetSession, f.scope.campaignId, evidence, + { directory, deadlineAt: Date.now() + 10_000 }) }; +} +describe("chat continuity journey controls", () => { + it("uses ordinary user messages once, owns the fresh chat before dispatch, and retains the independently graded result", async () => { + const f = await fixture(); + await f.run(); + expect(f.messages).toEqual([f.scenario.remember, f.scenario.recall, "/new", f.scenario.fresh]); + expect(f.actions.indexOf("intent")).toBeLessThan(f.actions.indexOf("send")); + expect(f.actions.indexOf("history")).toBeLessThan(f.actions.indexOf("intent")); + expect(f.own).toHaveBeenCalledTimes(2); + expect(f.own.mock.calls[0]![0]).toMatchObject({ journeyId: "chat-continuity", freshChat: { absent: true, userId: f.scope.userId } }); + expect(f.own.mock.calls[1]![0]).toMatchObject({ id: "chat", journeyId: "chat-continuity" }); + expect(f.evidence.checks).toHaveLength(9); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + expect(mocks.reply).toHaveBeenCalledTimes(5); + expect(mocks.capture).toHaveBeenCalledTimes(6); + expect(f.page.goto.mock.calls.flat().every(url => typeof url !== "string" || url.startsWith("https://qa.example.test/"))).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.map((point: { phase: string }) => point.phase)).toEqual(["initial", "returned", "recalled", "reset", "fresh", "reopened"]); + expect(saved.checks.every((check: { passed: boolean }) => check.passed)).toBe(true); + }); + it("accepts compact details with original reset summaries despite more than 500 unrelated company events", async () => { + const f = await fixture({ compactRuns: true, unrelatedActivity: 750 }); + await f.run(); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.every((point: { activity: unknown[] }) => point.activity.length === 0)).toBe(true); + expect(saved.points[5].resetReceipts[0]).toMatchObject({ source: "issue-runs", runId: "run-2", agentId: f.scope.agentId, + contextIssueId: "chat", wakeCommentId: "user-2", resultJson: { conversationReset: true } }); + expect(saved.points[5].runs[2].resultJson).toBeNull(); + }); + it.each([1, 750])("rejects an ignored agent filter instead of filtering %s unrelated global activity rows", async unrelatedActivity => { + const f = await fixture({ unrelatedActivity, ignoreActivityScope: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_observation_incomplete"); + expect(f.messages).toEqual([f.scenario.remember]); + }); + it("accepts board provisioning whose created entity is the owned agent, rather than treating its entity id as the actor", async () => { + const f = await fixture({ provisioningActor: "board", unrelatedActivity: 750 }); + expect(f.companyActivity.at(-1)).toMatchObject({ action: "agent.created", agentId: null, entityId: f.scope.agentId }); + await f.run(); + expect(f.evidence.checks.every(check => check.passed)).toBe(true); + const saved = JSON.parse(await readFile(path.join(f.directory, "chat-continuity.json"), "utf8")); + expect(saved.points.every((point: { activity: unknown[] }) => point.activity.length === 0)).toBe(true); + }); + it("rejects creation attributed to the fresh owned agent even if that activity predates the first send", async () => { + const f = await fixture({ provisioningActor: "owned-agent" }); + await expect(f.run()).rejects.toThrow("chat_continuity_evidence_failed"); + expect(f.evidence.checks.find(check => check.id === "chat-no-unrequested-work")?.passed).toBe(false); + }); + it.each(["succeeded", "running", "queued", "failed"])("rejects prior %s work on a reused agent before any send", async priorAgentStatus => { + const f = await fixture({ priorAgentStatus, unrelatedActivity: 750 }); + await expect(f.run()).rejects.toThrow("fresh_chat_agent_required"); + expect(f.messages).toEqual([]); + expect(f.own).not.toHaveBeenCalled(); + }); + it("rejects mixed or foreign agent history before any send", async () => { + const f = await fixture({ mixedAgentHistory: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_agent_history_scope_mismatch"); + expect(f.messages).toEqual([]); + expect(f.own).not.toHaveBeenCalled(); + }); + it.each(["preexisting", "intentFails"] as const)("does not dispatch model work when %s rejects ownership", async option => { + const f = await fixture({ [option]: true }); + await expect(f.run()).rejects.toThrow(); + expect(f.messages).toEqual([]); + expect(f.evidence.checks.every(check => !check.passed)).toBe(true); + }); + it("never replays an uncertain send", async () => { + const f = await fixture({ sendFails: true }); + await expect(f.run()).rejects.toThrow("uncertain send response"); + expect(f.messages).toEqual([f.scenario.remember]); + expect(mocks.send).toHaveBeenCalledTimes(1); + expect(f.evidence.checks.some(check => !check.passed)).toBe(true); + }); + it("rejects a foreign native run before sending another message", async () => { + const f = await fixture({ foreignRun: true }); + await expect(f.run()).rejects.toThrow("chat_continuity_run_mismatch"); + expect(f.messages).toEqual([f.scenario.remember]); + expect(f.evidence.checks.some(check => !check.passed)).toBe(true); + }); +}); diff --git a/tests/runner-e2e/product-journeys/chat-continuity-flow.ts b/tests/runner-e2e/product-journeys/chat-continuity-flow.ts new file mode 100644 index 00000000000..fe2a2301371 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity-flow.ts @@ -0,0 +1,164 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { sendChatMessage } from "../chat-flow.js"; +import { readBoundTaskFinalReply } from "../native-task-reply.js"; +import { collectRunEvents } from "../run-observations.js"; +import { chatContinuityActivityComplete, chatContinuityIssueId, chatContinuityResetReceiptMatches, + chatContinuityResetRun, chatContinuityScenario, gradeChatContinuity, + type ChatContinuityPoint, type ChatContinuityScope, type ChatContinuityResetReceipt } from "./chat-continuity-evidence.js"; +import { createObservedReader } from "./observation-read.js"; +import { captureProductScreenshot } from "./product-screenshot.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +interface Options { deadlineAt: number; directory: string } + +export async function runChatContinuity(context: JourneyContext, session: TargetSession, + campaignId: string, evidence: JourneyEvidence, options: Options) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !agent.model || !agent.provider || !context.environmentId) { + throw new JourneyError("unsupported", "native_agent_and_environment_required"); + } + const scope: ChatContinuityScope = { companyId: context.company.id, userId: context.user.id, + agentId: agent.id, agentName: agent.name, environmentId: context.environmentId, + provider: agent.provider, model: agent.model, campaignId }; + const scenario = chatContinuityScenario(campaignId), reader = createObservedReader(context.api, options.deadlineAt); + const chatPath = `/api/companies/${scope.companyId}/chats/${agent.id}`; + const chatUrl = `${context.api.origin}/${encodeURIComponent(context.company.issuePrefix)}/chats/${encodeURIComponent(agent.id)}`; + const points: ChatContinuityPoint[] = []; + let chatId: string | undefined, owned = false; + const remaining = () => { + const value = options.deadlineAt - Date.now(); + if (value <= 0) throw new JourneyError("failed", "chat_continuity_deadline"); + return value; + }; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = async () => writeFile(path.join(options.directory, "chat-continuity.json"), + JSON.stringify({ scope, points, checks: gradeChatContinuity(scope, points) }, null, 2), { mode: 0o600 }); + const freshChat = { companyId: scope.companyId, userId: scope.userId, leadAgentId: agent.id, absent: true as const }; + const read = async (): Promise | undefined> => { + const chat = await reader.get(chatPath); + if (!chat) return; + if (!chat.id || chat.companyId !== scope.companyId || chat.assigneeAgentId !== agent.id + || chat.conversationAgentId !== agent.id || chat.conversationUserId !== scope.userId + || chat.createdByUserId !== scope.userId || chat.title !== `Chat with ${agent.name}` || chatId && chat.id !== chatId) { + throw new JourneyError("failed", "chat_continuity_scope_mismatch"); + } + chatId ??= chat.id; + if (!owned) { + await session.own({ kind: "issue", name: chat.title, id: chat.id, journeyId: "chat-continuity", freshChat }); + owned = true; + } + const [summaries, comments, tasks, activity] = await Promise.all([ + reader.get(`/api/issues/${chat.id}/runs`), reader.get(`/api/issues/${chat.id}/comments?order=asc`), + reader.get(`/api/companies/${scope.companyId}/issues?view=index&assigneeAgentId=${agent.id}&includeAgentCreatedTasks=true&limit=2`), + reader.get(`/api/companies/${scope.companyId}/activity?agentId=${agent.id}&limit=500`), + ]); + if (![summaries, comments, tasks, activity].every(Array.isArray) || summaries.length > 4 || !chatContinuityActivityComplete(scope, activity) + || new Set(summaries.map(row => row.runId)).size !== summaries.length) { + throw new JourneyError("failed", "chat_continuity_observation_incomplete"); + } + const runs = await Promise.all(summaries.map(async row => { + if (typeof row.runId !== "string" || !row.runId) throw new JourneyError("failed", "chat_continuity_run_mismatch"); + if (row.agentId !== agent.id || row.contextIssueId !== chat.id) throw new JourneyError("failed", "chat_continuity_summary_scope_mismatch"); + const run = await reader.get(`/api/heartbeat-runs/${row.runId}`); + if (run.id !== row.runId || run.companyId !== scope.companyId || run.agentId !== agent.id + || chatContinuityIssueId(run) !== chat.id || !["queued", "running", "succeeded"].includes(run.status)) { + throw new JourneyError("failed", "chat_continuity_run_mismatch"); + } + return run; + })); + const resetReceipts: ChatContinuityResetReceipt[] = summaries.flatMap(summary => summary.resultJson?.conversationReset === true ? [{ + source: "issue-runs" as const, companyId: scope.companyId, issueId: chat.id, + runId: summary.runId, agentId: summary.agentId, status: summary.status, + contextIssueId: summary.contextIssueId, wakeCommentId: summary.wakeCommentId, resultJson: { conversationReset: true as const }, + }] : []); + return { chat, runs, comments, tasks, activity, resetReceipts }; + }; + const wait = async (providerCount: number, resetCount: number) => { + let stable = ""; + while (remaining() > 0) { + const state = await read(); + if (state) { + const providerRuns = state.runs.filter(run => !chatContinuityResetRun(run)); + const resetRuns = state.runs.filter(chatContinuityResetRun); + if (providerRuns.length > providerCount || resetRuns.length > resetCount) throw new JourneyError("failed", "chat_continuity_extra_run"); + const ready = providerRuns.length === providerCount && resetRuns.length === resetCount + && state.runs.every(run => run.status === "succeeded") + && providerRuns.every(run => run.nativePhase === "committed") + && resetRuns.every(run => state.resetReceipts.filter(receipt => chatContinuityResetReceiptMatches(scope, state.chat.id, run, receipt)).length === 1) + && state.chat.status === "in_review" && state.chat.conversationState === "waiting" + && !state.chat.executionRunId && !state.chat.scheduledRetry && !state.chat.activeRecoveryAction; + const key = ready ? JSON.stringify([state.chat.id, state.chat.conversationSessionGeneration, + state.runs.map(run => [run.id, run.status]), state.comments.map(row => row.id)]) : ""; + if (key && stable === key) return state; + stable = key; + } + await new Promise(resolve => setTimeout(resolve, Math.min(1000, remaining()))); + } + throw new JourneyError("failed", "chat_continuity_not_settled"); + }; + const snapshot = async (phase: ChatContinuityPoint["phase"], state: Awaited>, runId?: string) => { + let point: ChatContinuityPoint = { phase, ...state }; + if (runId) { + const load = async () => { + const latest = await read(); + const run = latest?.runs.find(run => run.id === runId); + if (!latest || !run) throw new JourneyError("failed", "chat_continuity_reply_run_missing"); + const events = await collectRunEvents((afterSeq, limit) => + reader.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + return { run, comments: latest.comments, events }; + }; + const reply = await readBoundTaskFinalReply({ page: context.page, runId, agentId: agent.id, + deadlineAt: options.deadlineAt, knownCompanyPrefixes: [context.company.issuePrefix], load }); + const latest = await read(), runEvidence = await load(); + if (!latest) throw new JourneyError("failed", "chat_continuity_reply_chat_missing"); + point = { phase, ...latest, reply, events: runEvidence.events }; + } + points.push(point); + await retain(); + await captureProductScreenshot(context.page, context.api.origin, path.join(options.directory, `chat-${phase}.png`), options.deadlineAt); + return point; + }; + const newProviderRun = (state: Awaited>, old: Row[] = []) => { + const added = state.runs.filter(run => !chatContinuityResetRun(run) && !old.some(before => before.id === run.id)); + if (added.length !== 1) throw new JourneyError("failed", "chat_continuity_turn_ambiguous"); + return added[0]!; + }; + try { + const settings = await reader.get("/api/instance/settings/experimental"); + if (settings.enableAgentChat !== true) throw new JourneyError("unsupported", "agent_chat_required"); + const history = await reader.get(`/api/companies/${scope.companyId}/heartbeat-runs?agentId=${agent.id}&limit=1&summary=true`); + if (!Array.isArray(history) || history.some(run => !run || run.companyId !== scope.companyId || run.agentId !== agent.id)) { + throw new JourneyError("failed", "chat_continuity_agent_history_scope_mismatch"); + } + if (history.length) throw new JourneyError("unsupported", "fresh_chat_agent_required"); + scope.agentHistory = { source: "company-agent-runs", companyId: scope.companyId, agentId: agent.id, limit: 1, runIds: [], observedBeforeSend: true }; + if (await reader.get(chatPath) !== null) throw new JourneyError("unsupported", "fresh_agent_chat_required"); + await session.own({ kind: "issue", name: `Chat with ${agent.name}`, journeyId: "chat-continuity", freshChat }); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(context.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); + await sendChatMessage(context.page, scenario.remember); + const first = await wait(1, 0), initial = await snapshot("initial", first, newProviderRun(first).id); + await context.page.goto(`${context.api.origin}/${encodeURIComponent(context.company.issuePrefix)}/issues`, { waitUntil: "domcontentloaded", timeout: remaining() }); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await context.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(context.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); + await snapshot("returned", await wait(1, 0), initial.reply!.runId); + await sendChatMessage(context.page, scenario.recall); + const second = await wait(2, 0), recalled = await snapshot("recalled", second, newProviderRun(second, initial.runs).id); + await sendChatMessage(context.page, scenario.reset); + await snapshot("reset", await wait(2, 1)); + await sendChatMessage(context.page, scenario.fresh); + const third = await wait(3, 1), freshRun = newProviderRun(third, recalled.runs); + await snapshot("fresh", third, freshRun.id); + await context.page.goto(chatUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); + await snapshot("reopened", await wait(3, 1), freshRun.id); + } finally { + evidence.checks.push(...gradeChatContinuity(scope, points)); + await retain(); + await reader.preserve(options.directory); + } + if (evidence.checks.some(check => !check.passed)) throw new JourneyError("failed", "chat_continuity_evidence_failed"); +} diff --git a/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts b/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts new file mode 100644 index 00000000000..a236106b597 --- /dev/null +++ b/tests/runner-e2e/product-journeys/chat-continuity.test-fixture.ts @@ -0,0 +1,66 @@ +import { chatContinuityScenario, type ChatContinuityPoint, type ChatContinuityScope } from "./chat-continuity-evidence.js"; + +type Row = Record; +export function chatContinuityFixture() { + const scope: ChatContinuityScope = { companyId: "company", userId: "board", agentId: "agent", agentName: "Owned agent", + environmentId: "environment", provider: "opencode", model: "configured-model", campaignId: "synthetic-continuity-123", + agentHistory: { source: "company-agent-runs", companyId: "company", agentId: "agent", limit: 1, runIds: [], observedBeforeSend: true } }; + const scenario = chatContinuityScenario(scope.campaignId); + const user = (id: string, body: string, generation: number) => ({ id, body, authorUserId: scope.userId, + authorAgentId: null, companyId: scope.companyId, issueId: "chat", conversationSessionGeneration: generation }); + const answer = (id: string, runId: string, body: string, generation: number) => ({ id, body, + authorAgentId: scope.agentId, authorUserId: null, createdByRunId: runId, + companyId: scope.companyId, issueId: "chat", conversationSessionGeneration: generation }); + const prompts = [scenario.remember, scenario.recall, scenario.reset, scenario.fresh]; + const board = prompts.map((body, index) => user(`user-${index}`, body, index >= 2 ? 1 : 0)); + const replies = [answer("reply-0", "run-0", "Remembered.", 0), answer("reply-1", "run-1", scenario.remembered, 0), + answer("reply-3", "run-3", scenario.marker, 1)]; + const run = (index: number): Row => ({ id: `run-${index}`, companyId: scope.companyId, agentId: scope.agentId, + nativeIssueId: "chat", status: "succeeded", runtimeMode: "native", nativePhase: "committed", + nativeSessionId: index === 3 ? "new-session" : "original-session", sessionIdBefore: null, + contextSnapshot: { issueId: "chat", source: "issue.comment", wakeCommentId: board[index]!.id, + conversationSessionGeneration: index >= 2 ? 1 : 0, paperclipTaskMarkdown: index === 3 ? scenario.fresh : scenario.remember, + paperclipEnvironment: { id: scope.environmentId } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } }, + resultJson: { finalizationPhase: "committed", presentationDecision: { commentId: `reply-${index}` } } }); + const runs = [run(0), run(1), run(2), run(3)]; + runs[2]!.contextSnapshot.conversationReset = true; + runs[2]!.resultJson = { conversationReset: true }; + delete runs[2]!.runnerProfileJson; + delete runs[2]!.nativeSessionId; + delete runs[2]!.nativePhase; + delete runs[2]!.runtimeMode; + const chat = { id: "chat", companyId: scope.companyId, assigneeAgentId: scope.agentId, conversationAgentId: scope.agentId, + conversationUserId: scope.userId, createdByUserId: scope.userId, title: `Chat with ${scope.agentName}`, + status: "in_review", conversationState: "waiting", conversationSessionGeneration: 0 }; + const comments = [board[0]!, replies[0]!, board[1]!, replies[1]!, board[2]!, board[3]!, replies[2]!]; + const makePoint = (phase: ChatContinuityPoint["phase"], count: number, commentCount: number, generation: number, replyIndex?: number): ChatContinuityPoint => { + const reply = replyIndex === undefined ? undefined : replies[replyIndex]!; + return structuredClone({ phase, chat: { ...chat, conversationSessionGeneration: generation }, runs: runs.slice(0, count), + comments: comments.slice(0, commentCount), tasks: [], activity: [], events: [], resetReceipts: count < 3 ? [] : [{ + source: "issue-runs", companyId: scope.companyId, issueId: "chat", runId: "run-2", agentId: scope.agentId, + status: "succeeded", contextIssueId: "chat", wakeCommentId: "user-2", resultJson: { conversationReset: true }, + }], + ...(reply ? { reply: { runId: reply.createdByRunId, agentId: scope.agentId, observedAfterReload: true, + body: reply.body, anchor: reply.id, visible: true, settled: true, nativeFinal: false } } : {}) }); + }; + const points = [makePoint("initial", 1, 2, 0, 0), makePoint("returned", 1, 2, 0, 0), makePoint("recalled", 2, 4, 0, 1), + makePoint("reset", 3, 5, 1), makePoint("fresh", 4, 7, 1, 2), makePoint("reopened", 4, 7, 1, 2)]; + return { scope, scenario, points }; +} + +export function compactChatContinuityFixture() { + const fixture = chatContinuityFixture(); + for (const point of fixture.points) { + for (const run of point.runs) run.resultJson = null; + if (!point.reply) continue; + point.reply.anchor = `${point.reply.runId}:turn`; + point.reply.nativeFinal = true; + point.events = [{ seq: 1, runId: point.reply.runId, companyId: fixture.scope.companyId, agentId: fixture.scope.agentId, + eventType: "item.completed", protocolSchemaVersion: 1, sourceEventId: "event", sourceInstanceId: "instance", sourceSeq: 1, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: point.reply.runId, sourceKind: "runner", + eventType: "item.completed", sourceEventId: "event", sourceInstanceId: "instance", sourceSeq: 1, + payload: { kind: "agentMessage", channel: "final", text: point.reply.body } } } }]; + } + return fixture; +} diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index a1bd122db02..6c7dc9c2930 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -18,6 +18,7 @@ import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; +import { runChatContinuity } from "./chat-continuity-flow.js"; import { runRepositoryContinuation } from "./repository-continuation-flow.js"; import { runScheduledRoutine } from "./routine-schedule-flow.js"; import { runInputFile } from "./input-file-flow.js"; @@ -164,6 +165,9 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", + "chat-continuity-flow.ts", + "chat-continuity-evidence.ts", + "../chat-cases.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", @@ -297,6 +301,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri await mkdir(directory, { recursive: true, mode: 0o700 }); await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); } }); + else if (id === "chat-continuity") await runChatContinuity(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else await runAgentJourney(id, ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { diff --git a/tests/runner-e2e/product-journeys/product-screenshot.test.ts b/tests/runner-e2e/product-journeys/product-screenshot.test.ts index 486b6c19f0e..97b8b5134db 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.test.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.test.ts @@ -13,12 +13,12 @@ function fixture(options: { mounted?: boolean; redirect?: string; initialPath?: return { page, locator, waitFor, screenshot }; } describe("product screenshot boundary", () => { - it.each(["/QA/inbox/unread", "/QA/routines"])("retains an authenticated journey failure on %s", async initialPath => { + it.each(["/QA/inbox/unread", "/QA/routines", "/QA/chats/owned-agent"])("retains an authenticated journey failure on %s", async initialPath => { const f = fixture({ initialPath }); await captureProductScreenshot(f.page, origin, "failed.png"); expect(f.screenshot).toHaveBeenCalledTimes(1); }); - it.each(["/login", "/QA/settings/keys", "/QA/inboxing", "/QA/routines-export"])("excludes unrelated or authentication paths: %s", async initialPath => { + it.each(["/login", "/QA/settings/keys", "/QA/inboxing", "/QA/routines-export", "/QA/chats-export"])("excludes unrelated or authentication paths: %s", async initialPath => { const f = fixture({ initialPath }); await expect(captureProductScreenshot(f.page, origin, "failed.png")).rejects.toThrow("product_screen_unavailable"); expect(f.waitFor).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/product-journeys/product-screenshot.ts b/tests/runner-e2e/product-journeys/product-screenshot.ts index cedfe14a17a..c7d352c4102 100644 --- a/tests/runner-e2e/product-journeys/product-screenshot.ts +++ b/tests/runner-e2e/product-journeys/product-screenshot.ts @@ -3,7 +3,7 @@ import { JourneyError } from "./types.js"; export function isProductScreenshotUrl(value: string, origin: string): boolean { const url = new URL(value); - return url.origin === origin && /^\/[^/]+\/(issues|search|inbox|routines)(\/|$)/.test(url.pathname); + return url.origin === origin && /^\/[^/]+\/(issues|search|inbox|routines|chats)(\/|$)/.test(url.pathname); } export async function captureProductScreenshot(page: Page, origin: string, file: string, deadlineAt = Date.now() + 10_000) { diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index d5d9a15b874..7369165693a 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -4,6 +4,7 @@ export const agentJourneyRuns = { "routine-scheduled-run": 1, "file-input-answer": 1, "ask-question": 1, + "chat-continuity": 4, "plan-revise-accept": 3, "build-revise": 2, "followup-while-running": 2, @@ -34,6 +35,7 @@ export const journeyRequiredChecks: Record = { "project-task-context": ["browser-created-project-persisted", "browser-created-project-task-persisted", "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"], "task-search-filter": ["task-search-results-exact", "task-status-filter-excludes-controls", "task-search-filter-reloaded", "task-search-result-reopened", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], + "chat-continuity": ["chat-fresh-owned-conversation", "chat-user-messages-stored-once", "chat-reload-preserved-context", "chat-new-session-boundary", "chat-fresh-context-answer", "chat-history-preserved-after-reload", "chat-no-unrequested-work", "chat-exact-turn-accounting", "configured-model-and-environment"], "question-answer-resume": ["question-resume.forms", "question-resume.choice.answer", "question-resume.choice.path", "question-resume.text.answer", "question-resume.text.path", "question-resume.complete-evidence", "question-no-premature-output", "question-answers-owned-by-board", "question-document-attributed", "question-document-reloaded", "configured-model-and-environment"], diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index eb79fd03411..cc92860f48b 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -9,6 +9,7 @@ export type JourneyId = | "project-task-context" | "task-document-persistence" | "ask-question" + | "chat-continuity" | "cloud-onboarding" | "question-answer-resume" | "task-search-filter"