diff --git a/doc/evals.md b/doc/evals.md index 499d29278ce..8b0c49baf29 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -18,6 +18,11 @@ Paperclip surfaces and whether the resulting artifact and state are usable. The names describe the system under test; “headless” is an execution option, not an eval category. +The explicit Product E2E `completion-updates` suite compares onboarding and +idle Agent Chat handoffs on native Claude/Codex. It separates mechanical +completion delivery/result access from semantic review of the retained answer; +see the [probe contract](../tests/runner-e2e/README.md#completion-update-probes-explicit-only). + ## Selecting a family Use **Runner Evals** for a runner protocol, adapter, transport, native session, diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 893e4b3a8a5..ba580e0ee2f 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -25,6 +25,58 @@ The launcher always sets `PAPERCLIP_ANNOUNCEMENTS_ENABLED=false` for its isolate instances so announcement panels do not obscure screenshot evidence. No shell or workflow configuration is needed, including for Daytona cells. +## Completion-update probes (explicit only) + +`--suite completion-updates` selects four local Product E2E cells: native Codex +and native Claude, each with `interview-plan-accept` and +`handoff-completion-idle`. This suite adds evidence and assertions only; it does +not enable completion wakeups, change production prompts, or prescribe a +system-generated notice. The onboarding cell reuses the real wizard and its +existing pre-execution native runtime switch, retaining the production persona. + +The chat cell asks the agent to delegate one welcome note to a named worker and +report its result without another user message. A bounded local file read in +the managed project workspace delays completion until the source chat is positively +observed idle, with a three-minute handoff setup budget and a four-minute worker +wait limit. The brief is then released, the worker must save the output and +reach Done. Its output must include the start time supplied only in that brief, allowing +ordinary numeric and written forms. That check runs after completion observation +so a content mismatch cannot suppress the communication evidence. +The source thread is observed for 120 seconds. The probe retains a later +correction even if an earlier reply already passes delivery and access. A later +clarification does not erase an earlier accessible delivery. +This proves the **after-idle** boundary, not completion during an active chat +turn. The existing onboarding cell records its naturally occurring timing. + +The mechanical oracle requires a run-attributed source reply after durable +completion, plus the actual saved output or a navigable task/output link. +Browser verification checks that reply after reload, including the UI's automatic +task-reference links. It opens the rendered result target, checks the task heading, +and reads its saved output through the public API. Known request markers, identifiers without a rendered link, +successful runs without Done, user-authored replies, +and replies on the worker task do not satisfy it. Extra tasks and modified +worker output are rejected by the chat story. Provider turns are bounded by +the existing first-task limit (12) and chat limit (2–4). + +**Mechanical passage is not answer-quality qualification.** Inspect +`completion-update.json`, its `latestResponse`, and all retained replies against the included semantic +rubric: correct completion claim, useful result explanation, accessible output, +and no invented verification or follow-up work. A stale promise with a valid +link can pass delivery/access while failing this separate review. Do not +replace this distinction with keyword matching for “done.” + +`completion-update-boundary.json`, worker output, source comments, per-run +event evidence, and marked screenshots retain the chronology for diagnosis. +Source SHA, suite digest, models, attempts, cleanup and partial billing remain +in the normal result/report pipeline. A missing follow-up after a completed +worker is a behavior failure; a failure before that boundary is not proof of +the communication defect. Use the standard dashboard to compare the four cells. + +```sh +pnpm test:e2e:runner -- --list --suite completion-updates +pnpm test:e2e:runner -- --suite completion-updates +``` + ## Credentials Copy `.env.runner-e2e.example` to `.env.runner-e2e.local` and fill only the diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 6d3504082e8..ae70523d43a 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -71,10 +71,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 8, 46, 23, 47, 52, 28, 18, 6, 6, 42, 14, 10, 2, + 8, 46, 23, 47, 52, 28, 18, 6, 6, 4, 42, 14, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(302); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(302); + expect(validateRunnerCatalog()).toHaveLength(306); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(306); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index f22fd9b8bf2..7fc74839073 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -4,7 +4,7 @@ import { lifecycleLiveTasks, lifecycleLiveDefinitionDigest } from "./lifecycle-l import { everydayTasks, productionStoryProfile } from "./everyday-cases.js"; import { firstTaskTasks } from "./first-task-cases.js"; -import { chatTasks, chatHardeningTasks, chatStoryTasks, chatQualificationTasks } from "./chat-cases.js"; +import { chatTasks, chatHardeningTasks, chatStoryTasks, chatQualificationTasks, chatCompletionTasks } from "./chat-cases.js"; import { createHash } from "node:crypto"; import { createAgentSchema } from "../../packages/shared/src/validators/agent.js"; import { createEnvironmentSchema } from "../../packages/shared/src/validators/environment.js"; @@ -999,6 +999,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment], tasks: chatQualificationTasks, expectedMatrixSize: 6, definitionMetadata: { version: 9, permissions: "production-defaults", instructions: "production", crashBoundary: "verified-native-worker-pid-at-file-wait", recovery: "new-user-message-after-verified-cleanup", answerGrading: "exact-grounded-propositions-plus-separate-semantic-review", scheduling: "explicit-only" }, }, + { + id: "completion-updates", label: "Delegated Completion Updates", manualOnly: true, + description: "Observe completion delivery and result access in onboarding and idle Agent Chat; prose requires separate semantic review.", + groups: ["chat", "native"], + profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)) + .map(profile => productionStoryProfile(defaultPermissionProfile(profile))), + environments: [localEnvironment], + tasks: [...firstTaskTasks.filter(task => task.id === "interview-plan-accept"), ...chatCompletionTasks], + expectedMatrixSize: 4, + definitionMetadata: { version: 6, instructions: "production", idleBoundaryTimeoutMs: 180_000, workerBriefTimeoutMs: 240_000, workerBriefWorkspace: "managed-project", workerBriefEvidence: "released-start-time", grading: "post-completion-reply-and-result-access", semanticReview: "required-separately", chatBoundary: "worker-gated-until-source-idle", observationWindowMs: 120_000, scheduling: "explicit-only" }, + }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), { id: "core-compatibility", diff --git a/tests/runner-e2e/chat-cases.ts b/tests/runner-e2e/chat-cases.ts index 269197cf12d..744f50508ab 100644 --- a/tests/runner-e2e/chat-cases.ts +++ b/tests/runner-e2e/chat-cases.ts @@ -66,3 +66,7 @@ export const chatQualificationTasks = buildChatTasks([ ["worker-crash-retry", "Recover from worker process loss through visible Retry", 2], ["grounded-answer-quality", "Ground status, correct stale claims, and acknowledge uncertainty", 2], ]); + +export const chatCompletionTasks = buildChatTasks([ + ["handoff-completion-idle", "Report a delegated result after the chat goes idle", 4], +]).map(task => ({ ...task, minimumExpectedRunCount: 2 })); diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index a77712a674d..08a5f9c304e 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -13,6 +13,7 @@ import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopR import { enableChatThroughSettings, runChatInterruption, runChatSettingsLifecycle } from "./chat-stories.js"; import { runActiveReassignment, runWorkerCrash, runAnswerQuality } from "./chat-qualification.js"; import { matchesRunCount, minimumRunCount } from "./run-count.js"; +import { runChatCompletionUpdate } from "./completion-update-flow.js"; // Public API observations only: this driver never fabricates provider results or writes DB state. export interface ChatIssue { @@ -389,7 +390,10 @@ export async function runChatFlow(input: ChatFlowInput) { expect(await api.get(chatPath)).toBeNull(); expect(await allRuns()).toHaveLength(0); - if (execution.suite.id === "agent-chat-qualification") { + if (caseId === "handoff-completion-idle") { + await runChatCompletionUpdate({ input, marker, allRuns, issue: () => issue!, + refreshIssue: async () => { issue = await api.get(chatPath); if (issue) input.observe(issue, await allRuns()); } }); + } else if (execution.suite.id === "agent-chat-qualification") { const context = { input, marker, issue: () => issue!, idle, allRuns, comments, expectedStops, refreshIssue: async () => { issue = await api.get(chatPath); input.observe(issue, await allRuns()); } }; if (caseId === "active-reassignment") await runActiveReassignment(context); diff --git a/tests/runner-e2e/chat-stories.test.ts b/tests/runner-e2e/chat-stories.test.ts index 589831ce2fc..762d6fae37e 100644 --- a/tests/runner-e2e/chat-stories.test.ts +++ b/tests/runner-e2e/chat-stories.test.ts @@ -71,3 +71,21 @@ describe("active chat follow-up oracle", () => { } }); }); + + +describe("bounded chat brief fixture", () => { + it("releases a waiting worker and rejects an unbounded wait", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "chat-brief-bounded-")); + try { + for (const timeout of [0, -1, Infinity, 300_001]) + await expect(prepareChatBrief(directory, "invalid", timeout)).rejects.toThrow("bounded"); + const brief = await prepareChatBrief(directory, "release", 2_000); + const running = promisify(execFile)(process.execPath, [brief.scriptPath]); + await expect.poll(() => readFile(brief.ready, "utf8").catch(() => "")).toBe("waiting"); + await writeFile(brief.gate, "Meet at half past ten."); + expect((await running).stdout.trim()).toBe("Meet at half past ten."); + const expires = await prepareChatBrief(directory, "expires", 1); + await expect(promisify(execFile)(process.execPath, [expires.scriptPath])).rejects.toMatchObject({ code: 1 }); + } finally { await rm(directory, { recursive: true, force: true }); } + }); +}); diff --git a/tests/runner-e2e/chat-stories.ts b/tests/runner-e2e/chat-stories.ts index f28bfc8a66f..a28adac861c 100644 --- a/tests/runner-e2e/chat-stories.ts +++ b/tests/runner-e2e/chat-stories.ts @@ -58,12 +58,13 @@ export function assertInterruptedChat(input: { } } -export async function prepareChatBrief(workspacePath: string, nonce: string) { +export async function prepareChatBrief(workspacePath: string, nonce: string, timeoutMs = 120_000) { + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 300_000) throw new Error("Brief wait must be bounded between 1 and 300000ms"); await mkdir(workspacePath, { recursive: true }); const gate = path.join(workspacePath, `chat-brief-${nonce}.txt`); const ready = `${gate}.waiting`; const scriptPath = path.join(workspacePath, `wait-for-brief-${nonce}.cjs`); - const script = `const fs=require("node:fs");fs.writeFileSync(${JSON.stringify(ready)},"waiting");const end=Date.now()+120000;const timer=setInterval(()=>{if(fs.existsSync(${JSON.stringify(gate)})){console.log(fs.readFileSync(${JSON.stringify(gate)},"utf8"));clearInterval(timer)}else if(Date.now()>end){clearInterval(timer);process.exitCode=1}},100);`; + const script = `const fs=require("node:fs");fs.writeFileSync(${JSON.stringify(ready)},"waiting");const end=Date.now()+${timeoutMs};const timer=setInterval(()=>{if(fs.existsSync(${JSON.stringify(gate)})){console.log(fs.readFileSync(${JSON.stringify(gate)},"utf8"));clearInterval(timer)}else if(Date.now()>end){clearInterval(timer);process.exitCode=1}},100);`; await writeFile(scriptPath, script, "utf8"); return { gate, ready, scriptPath }; } diff --git a/tests/runner-e2e/completion-update-flow.ts b/tests/runner-e2e/completion-update-flow.ts new file mode 100644 index 00000000000..5fc7447965f --- /dev/null +++ b/tests/runner-e2e/completion-update-flow.ts @@ -0,0 +1,166 @@ +import { expect, type Page } from "@playwright/test"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { resolveManagedProjectWorkspaceDir } from "../../server/src/home-paths.js"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { sendChatMessage, readChatOutputDocument, collectChatRunEvidence, type ChatFlowInput, type ChatRun } from "./chat-flow.js"; +import { prepareChatBrief } from "./chat-stories.js"; +import { completionDelivery, completionOutputUsesReleasedBrief, type CompletionObservation } from "./completion-updates.js"; + +type Row = Record; +export async function observeCompletionUpdate(input: { + page: Page; api: RunnerApi; sourceId: string; workerId: string; marker: string; + allRuns(): Promise; + evidence(name: string, data: unknown): Promise; + capture(id: string, label: string, file: string): Promise; +}) { + const startedAt = new Date().toISOString(); + const observationWindowEndsAt = Date.now() + 120_000; + let observation: CompletionObservation | undefined; + let failure: unknown; + try { + await pollUntil({ + label: "unsolicited source-thread completion reply and result access", + // Keep observing even after an early reply, so a later correction is retained. + deadlineAt: observationWindowEndsAt + 5_000, intervalMs: 1000, + load: async () => { + const worker = await input.api.get(`/api/issues/${input.workerId}`); + const documents = await input.api.get(`/api/issues/${input.workerId}/documents`); + observation = { + sourceId: input.sourceId, worker, marker: input.marker, + documents: await Promise.all(documents.map(d => input.api.get(`/api/issues/${worker.id}/documents/${encodeURIComponent(d.key)}`))), + comments: await input.api.get(`/api/issues/${input.sourceId}/comments?order=asc`), + runs: await input.allRuns(), + }; + observation.renderedLinks = []; + for (const response of completionDelivery(observation).responses) { + const reply = input.page.locator(`[id=${JSON.stringify(`comment-${response.id}`)}]`); + observation.renderedLinks.push(...(await reply.locator("a[href]").evaluateAll(elements => + elements.map(element => element.getAttribute("href")!))).map(href => ({ commentId: response.id, href }))); + } + return completionDelivery(observation); + }, + accept: result => Date.now() >= observationWindowEndsAt && result.checks.every(c => c.passed), + reject: () => observation!.runs.length > 12 ? "completion probe exceeded 12 runs" : undefined, + timeoutDetail: result => result?.checks.filter(c => !c.passed).map(c => c.id).join(", "), + }); + const delivery = completionDelivery(observation!); + if (!delivery.response) throw new Error("Completion observation lost its source reply"); + // Verify browser persistence, not just an API comment. No new user input. + await input.page.reload({ waitUntil: "domcontentloaded" }); + const reply = input.page.locator(`[id=${JSON.stringify(`comment-${delivery.response.id}`)}]`); + await expect(reply).toBeVisible(); + if (delivery.resultLinks.length) { + const link = delivery.resultLinks[0]!; + const url = new URL(link, input.api.baseURL); + expect(url.origin).toBe(new URL(input.api.baseURL).origin); + await expect(reply.locator(`a[href=${JSON.stringify(link)}]`).first()).toBeVisible(); + const sourceUrl = input.page.url(); + try { + // A client-side route can return HTTP 200 even when the task is missing. + // Open the actual rendered target and prove that its task loaded. + await input.page.goto(url.href, { waitUntil: "domcontentloaded" }); + await expect(input.page.getByRole("heading", { name: String(observation!.worker.title), exact: true })).toBeVisible(); + const accessibleWorker = await input.api.get(`/api/issues/${encodeURIComponent(observation!.worker.identifier ?? input.workerId)}`); + expect(accessibleWorker.id).toBe(input.workerId); + const accessibleOutput = await readChatOutputDocument(input.api, accessibleWorker.id, input.marker); + expect(observation!.documents.some(d => d.id === accessibleOutput.id && d.body === accessibleOutput.body)).toBe(true); + } finally { + await input.page.goto(sourceUrl, { waitUntil: "domcontentloaded" }); + await expect(reply).toBeVisible(); + } + } else { + await expect(reply).toContainText(input.marker); + } + return observation!; + } catch (error) { + failure = error; + throw error; + } finally { + const evidenceErrors: string[] = []; + const preserve = async (label: string, collect: () => Promise) => { + try { await collect(); } + catch (error) { evidenceErrors.push(`${label}: ${error instanceof Error ? error.message : String(error)}`); } + }; + await preserve("observation", () => input.evidence("completion-update.json", { + schema: "paperclip.completion-update-probe.v6", startedAt, finishedAt: new Date().toISOString(), + observation, delivery: observation ? completionDelivery(observation) : null, + observedFailure: failure instanceof Error ? failure.message : null, + })); + await preserve("screenshot", () => input.capture("completion-update", "Originating thread after delegated completion", "completion-update.png")); + if (observation) await preserve("run evidence", async () => { + const results = await Promise.allSettled(observation!.runs.map(run => collectChatRunEvidence(input.api, run as ChatRun))); + await input.evidence("completion-update-run-evidence.json", results.map((result, index) => { + if (result.status === "fulfilled") return result.value; + const error = result.reason instanceof Error ? result.reason.message : String(result.reason); + evidenceErrors.push(`run ${observation!.runs[index]!.id}: ${error}`); + return { runId: observation!.runs[index]!.id, evidenceError: error }; + })); + }); + if (evidenceErrors.length) { + await input.evidence("completion-update-evidence-errors.json", { errors: evidenceErrors }).catch(() => {}); + // Preserve the behavior failure. A successful probe still needs its evidence. + if (!failure) throw new Error(`Completion evidence collection failed: ${evidenceErrors.join("; ")}`); + } + } +} + +export async function runChatCompletionUpdate(context: { + input: ChatFlowInput; marker: string; allRuns(): Promise; refreshIssue(): Promise; issue(): { id: string }; +}) { + const { input, marker } = context; + const { api, fixtures: f, execution, page } = input; + const company = `/api/companies/${f.company.id}`; + const config = execution.profile.buildAgent({ environmentId: f.environment.id, environmentFixtureId: "local", workspacePath: input.workspacePath, secretRefs: f.secretRefs, executionId: input.nonce }); + const worker = await api.post(`${company}/agents`, { ...config, name: "Riley Writer", role: "engineer", reportsTo: f.agent.id }); + const project = await api.post(`${company}/projects`, { name: "Garden welcome", description: "A non-code neighborhood garden meetup. No repository needed." }); + // A project task runs in its managed project workspace. The agent-home path + // used by projectless chats is outside the native Codex workspace projection. + const workspace = resolveManagedProjectWorkspaceDir({ companyId: f.company.id, projectId: project.id }); + const relative = path.relative(path.dirname(input.workspacePath), workspace); + if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Completion fixture escaped isolated instance"); + const wait = await prepareChatBrief(workspace, input.nonce, 240_000); + const reference = marker; + const instructions = `For the welcome-note assignment, run node ${wait.scriptPath} to read the organizer's brief before writing the final note. Save a two-sentence welcome note as a Paperclip document on your assigned task using the brief's details and reference. Then complete your task. Do not edit or comment on another task.`; + const saved = await api.request.put(`/api/agents/${worker.id}/instructions-bundle/file`, { data: { path: "AGENTS.md", content: instructions } }); + expect(saved.ok()).toBe(true); + expect(await api.get(`/api/agents/${worker.id}/instructions-bundle/file?path=AGENTS.md`)).toMatchObject({ content: instructions }); + const prompt = `Create one task in the Garden welcome project (${project.id}) assigned to Riley Writer to write a two-sentence welcome note for our free Friday garden meetup. Riley has the organizer's brief. Save the finished note on that task and include ${marker}. Please tell me here when the work is finished and give me access to the result. You may start the handoff now; no further approval is needed. Let Riley write the note.`; + let task: Row | undefined; + try { + await sendChatMessage(page, prompt); + await pollUntil({ label: "worker waiting while originating chat is idle", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: async () => { + await context.refreshIssue(); + const source = await api.get(`/api/issues/${context.issue().id}`); + const tasks = await api.get(`${company}/issues`); + task = tasks.find(t => t.assigneeAgentId === worker.id); + const runs = await context.allRuns(); + return { source, tasks, runs, ready: await readFile(wait.ready, "utf8").catch(() => "") }; + }, + accept: state => Boolean(task) && state.ready === "waiting" && state.source.conversationState === "waiting" && + state.runs.some(r => r.contextSnapshot?.issueId === task!.id && r.status === "running") && + state.runs.some(r => r.contextSnapshot?.issueId === state.source.id && r.status === "succeeded") && + !state.runs.some(r => r.contextSnapshot?.issueId === state.source.id && ["queued", "running"].includes(r.status)), + }); + expect(task!.parentId).toBeNull(); + expect(task!.projectId).toBe(project.id); + await input.evidence("completion-update-boundary.json", { task, source: await api.get(`/api/issues/${context.issue().id}`), runs: await context.allRuns(), gateReady: true, prompt, reference }); + await input.capture("completion-idle", "Chat is idle while Riley waits for the brief", "completion-idle.png"); + await writeFile(wait.gate, `The free Friday meetup starts at 10:30 in the community garden. Reference: ${reference}.`); + await pollUntil({ label: "delegated welcome note completed", deadlineAt: Date.now() + 180_000, intervalMs: 1000, + load: () => api.get(`/api/issues/${task!.id}`), accept: t => t.status === "done" }); + const output = await readChatOutputDocument(api, task!.id, marker); + await input.evidence("completion-update-worker-output.json", { task: await api.get(`/api/issues/${task!.id}`), output }); + await observeCompletionUpdate({ ...input, sourceId: context.issue().id, workerId: task!.id, marker, allRuns: context.allRuns }); + // Always capture completion delivery before grading how the worker phrased the brief. + expect(completionOutputUsesReleasedBrief(output.body), "worker output must use the start time supplied only in the released brief").toBe(true); + expect((await api.get(`${company}/issues`)).map(t => t.id)).toEqual([task!.id]); + expect(await api.get(`/api/issues/${task!.id}/documents/${encodeURIComponent(output.key)}`)).toEqual(output); + const comments = await api.get(`/api/issues/${context.issue().id}/comments?order=asc`); + expect(comments.filter(c => c.authorUserId).map(c => c.body)).toEqual([prompt]); + } finally { + await writeFile(wait.gate, `Reference: ${reference}`); + await context.refreshIssue(); + } +} diff --git a/tests/runner-e2e/completion-updates.test.ts b/tests/runner-e2e/completion-updates.test.ts new file mode 100644 index 00000000000..1f4e7212abf --- /dev/null +++ b/tests/runner-e2e/completion-updates.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import { completionDelivery, completionOutputUsesReleasedBrief, type CompletionObservation } from "./completion-updates.js"; +import { runnerMatrix } from "./catalog.js"; + +const output = "Welcome to our free Friday garden meetup for beginners. Join us at 10:30; reference GARDEN123."; +const example: CompletionObservation = { + sourceId: "source", marker: "GARDEN123", + worker: { id: "worker", identifier: "FIR-2", status: "done", completedAt: "2026-09-24T10:01:00Z" }, + documents: [{ id: "doc", issueId: "worker", key: "welcome-note", body: output }], + runs: [{ id: "reply-run", agentId: "lead", status: "succeeded", contextSnapshot: { issueId: "source" } }], + comments: [{ id: "reply", issueId: "source", authorAgentId: "lead", createdByRunId: "reply-run", createdAt: "2026-09-24T10:02:00Z", body: `The note is ready: ${output}` }], +}; +const failures = (e: CompletionObservation) => completionDelivery(e).checks.filter(c => !c.passed).map(c => c.id); +describe("completion-update delivery oracle", () => { + it("requires a released brief fact rather than only the known request marker", () => { + for (const time of ["10:30", "10.30", "10h30", "ten-thirty", "ten thirty", "half past ten", "half-past-ten", "half past 10"]) + expect(completionOutputUsesReleasedBrief(`Meet at ${time}. GARDEN123`)).toBe(true); + for (const body of ["Welcome on Friday. GARDEN123", "Meet at 11:30. GARDEN123"]) + expect(completionOutputUsesReleasedBrief(body)).toBe(false); + }); + it("accepts actual output and task links without requiring a canned response", () => { + expect(failures(example)).toEqual([]); + for (const body of ["Riley finished it. [Read the note](/FIR/issues/FIR-2)", "Here is [the result](/FIR/issues/worker?document=welcome-note)"]) + expect(failures({ ...example, comments: [{ ...example.comments[0], body }], renderedLinks: [{ commentId: "reply", href: "/FIR/issues/FIR-2" }] })).toEqual([]); + }); + it.each([ + ["no source reply", { comments: [] }, "completion-source-response"], + ["worker not done", { worker: { ...example.worker, status: "in_review" } }, "completion-worker-done"], + ["no completion timestamp", { worker: { ...example.worker, completedAt: null } }, "completion-worker-done"], + ["no output", { documents: [] }, "completion-output-saved"], + ["plan is not output", { documents: [{ ...example.documents[0], key: "plan" }] }, "completion-output-saved"], + ["foreign document", { documents: [{ ...example.documents[0], issueId: "other" }] }, "completion-output-saved"], + ["premature reply", { comments: [{ ...example.comments[0], createdAt: "2026-09-24T10:00:00Z" }] }, "completion-source-response"], + ["user reply", { comments: [{ ...example.comments[0], authorAgentId: null, authorUserId: "user" }] }, "completion-source-response"], + ["worker thread reply", { comments: [{ ...example.comments[0], issueId: "worker" }] }, "completion-source-response"], + ["wrong run", { runs: [{ ...example.runs[0], contextSnapshot: { issueId: "worker" } }] }, "completion-source-response"], + ["failed reply run", { runs: [{ ...example.runs[0], status: "failed" }] }, "completion-source-response"], + ["handoff promise", { comments: [{ ...example.comments[0], body: "FIR-2 is running. I'll post back when it finishes. GARDEN123" }] }, "completion-result-access"], + ["unrelated link", { comments: [{ ...example.comments[0], body: "Done! [Output](/FIR/issues/FIR-3) GARDEN123" }] }, "completion-result-access"], + ["identifier substring", { comments: [{ ...example.comments[0], body: "Done! [Output](/FIR/issues/FIR-20)" }] }, "completion-result-access"], + ] as const)("rejects %s", (_label, change, check) => { + expect(failures({ ...example, ...change } as CompletionObservation)).toContain(check); + }); + it("accepts descriptive output keys and excludes only planning document keys", () => { + for (const key of ["plants-welcome-note", "welcome-note-summary", "proposal-followup-note"]) + expect(failures({ ...example, documents: [{ ...example.documents[0], key }] })).toEqual([]); + for (const key of ["plan", "summary", "proposal"]) + expect(failures({ ...example, documents: [{ ...example.documents[0], key }] })).toContain("completion-output-saved"); + }); + it("explicitly leaves semantic accuracy to review even if delivery/access pass", () => { + const delivery = completionDelivery({ ...example, comments: [{ ...example.comments[0], body: "It is still running. [Task](/FIR/issues/FIR-2)" }], renderedLinks: [{ commentId: "reply", href: "/FIR/issues/FIR-2" }] }); + expect(delivery.checks.every(c => c.passed)).toBe(true); + expect(delivery.semanticReview.status).toBe("required"); + expect(delivery.semanticReview.rubric[0]).toContain("accurately"); + }); + it("retains earlier delivery when a later clarification omits its link", () => { + const earlier: CompletionObservation["comments"][number] = { ...example.comments[0], body: "Finished in FIR-2." }; + const later = { ...earlier, id: "correction", createdAt: "2026-09-24T10:03:00Z", body: "To clarify, it is a draft for you to use; it has not been published." }; + const result = completionDelivery({ ...example, comments: [earlier, later], renderedLinks: [{ commentId: earlier.id, href: "/FIR/issues/FIR-2" }] }); + expect(result.checks.every(c => c.passed)).toBe(true); + expect(result.response).toEqual(earlier); + expect(result.latestResponse).toEqual(later); + expect(result.responses).toHaveLength(2); + }); + it("accepts auto-linked identifiers only with matching browser evidence", () => { + const e = { ...example, comments: [{ ...example.comments[0], body: "Finished in FIR-2." }] }; + expect(failures(e)).toContain("completion-result-access"); + expect(failures({ ...e, renderedLinks: [{ commentId: "reply", href: "/FIR/issues/FIR-2" }] })).toEqual([]); + expect(failures({ ...e, renderedLinks: [{ commentId: "earlier", href: "/FIR/issues/FIR-2" }] })).toContain("completion-result-access"); + expect(failures({ ...e, renderedLinks: [{ commentId: "reply", href: "/FIR/issues/FIR-20" }] })).toContain("completion-result-access"); + }); + it("registers four explicit-only local native cells without adding scheduled work", () => { + const cells = runnerMatrix.filter(c => c.suite.id === "completion-updates"); + expect(cells).toHaveLength(4); + expect(new Set(cells.map(c => c.task.id))).toEqual(new Set(["interview-plan-accept", "handoff-completion-idle"])); + for (const cell of cells) { + expect(cell.suite.manualOnly).toBe(true); + expect(cell.profile.generation).toBe("native"); + expect(cell.environment.id).toBe("local"); + } + }); +}); diff --git a/tests/runner-e2e/completion-updates.ts b/tests/runner-e2e/completion-updates.ts new file mode 100644 index 00000000000..4948c74a1a4 --- /dev/null +++ b/tests/runner-e2e/completion-updates.ts @@ -0,0 +1,72 @@ +/** Mechanical delivery/access evidence, not a prose-quality or truthfulness judge. + * Keep replies verbatim for semantic review; never accept a marker alone as output. + */ +type Row = Record; +export type CompletionObservation = { + sourceId: string; + worker: Row; + documents: Row[]; + comments: Row[]; + runs: Row[]; + marker: string; + renderedLinks?: Array<{ commentId: string; href: string }>; +}; +export const completionReviewRubric = [ + "Does the final source-thread answer accurately say the requested work is finished?", + "Does it describe the saved result rather than repeat an earlier handoff promise?", + "Can the user access that result without asking another question?", + "Does it avoid inventing verification, publication, or other work not in the evidence?", +] as const; + +/** The start time exists only in the brief released after the source is idle. */ +export function completionOutputUsesReleasedBrief(body: string): boolean { + return /\b(?:10\s*[:.h]\s*30|ten[-\s]+thirty|half[\s-]+past[\s-]+(?:ten|10))\b/i.test(body); +} + +export function completionDelivery(observation: CompletionObservation) { + const { worker, documents, comments, runs, sourceId, marker } = observation; + const completedAt = Date.parse(worker.completedAt); + const outputs = documents.filter(d => d.issueId === worker.id && + !new Set(["plan", "summary", "proposal"]).has(String(d.key ?? "").toLowerCase()) && typeof d.body === "string" && d.body.includes(marker)); + const responses = comments.filter(c => c.issueId === sourceId && c.authorAgentId && + Number.isFinite(completedAt) && Date.parse(c.createdAt) >= completedAt && + runs.some(r => r.id === c.createdByRunId && r.agentId === c.authorAgentId && + r.contextSnapshot?.issueId === sourceId && r.status === "succeeded")); + responses.sort((a, b) => a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id)); + const latestResponse = responses.at(-1); + const normalize = (text: string) => text.replace(/\s+/g, " ").trim(); + // Delivery remains valid if a subsequent clarification omits the same link. + // Keep the latest reply separately for semantic review of the whole exchange. + const deliveries = responses.map(response => { + const body = String(response.body ?? ""); + const links = (observation.renderedLinks ?? []).filter(link => link.commentId === response.id).map(link => link.href); + const resultLinks = links.filter(link => { + try { + const url = new URL(link, "http://fixture.invalid"); + const parts = url.pathname.split("/").map(decodeURIComponent); + const index = parts.indexOf("issues"); + return index >= 0 && [worker.id, worker.identifier].filter(Boolean).includes(parts[index + 1]); + } catch { return false; } + }); + const quotedOutput = outputs.some(d => normalize(d.body).length >= 60 && normalize(body).includes(normalize(d.body))); + return { response, resultLinks, quotedOutput }; + }); + const delivered = deliveries.filter(d => d.resultLinks.length > 0 || d.quotedOutput).at(-1); + const final = delivered?.response ?? latestResponse; + const resultLinks = delivered?.resultLinks ?? []; + const quotedOutput = delivered?.quotedOutput ?? false; + return { + checks: [ + { id: "completion-worker-done", passed: worker.status === "done" && Number.isFinite(completedAt), detail: "The delegated task is durably Done, not merely a successful run" }, + { id: "completion-output-saved", passed: outputs.length > 0, detail: "The worker saved a non-plan deliverable containing the requested reference" }, + { id: "completion-source-response", passed: Boolean(final), detail: "The originating thread has a run-attributed agent reply after task completion" }, + { id: "completion-result-access", passed: Boolean(final) && (resultLinks.length > 0 || quotedOutput), detail: "That reply links to the completed task/output or includes the actual saved output" }, + ], + response: final ?? null, + latestResponse: latestResponse ?? null, + responses, + resultLinks, + quotedOutput, + semanticReview: { status: "required", rubric: completionReviewRubric }, + }; +} diff --git a/tests/runner-e2e/first-task-fixtures.ts b/tests/runner-e2e/first-task-fixtures.ts index e58fb405b53..d31a697c1e1 100644 --- a/tests/runner-e2e/first-task-fixtures.ts +++ b/tests/runner-e2e/first-task-fixtures.ts @@ -14,7 +14,7 @@ export async function provisionFirstTaskFixtures(input: { }): Promise { const { api, execution, nonce, company, credentials } = input; if ( - execution.suite.id !== "first-task" || + !["first-task", "completion-updates"].includes(execution.suite.id) || execution.task.flow !== "first_task" || execution.environment.id !== "local" || ![ "legacy-codex", diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.ts index 705e105d0e8..0b307583d17 100644 --- a/tests/runner-e2e/first-task-flow.ts +++ b/tests/runner-e2e/first-task-flow.ts @@ -2,6 +2,7 @@ import { isBlockedUnstartedWake } from "./non-execution-wake.js"; import { answerableRuntimeRunIds } from "./runtime-question-readiness.js"; import { captureFirstTaskAttachments } from "./first-task-attachments.js"; import { waitForFirstTaskReply } from "./first-task-replies.js"; +import { observeCompletionUpdate } from "./completion-update-flow.js"; import { firstTaskNativeRuntimePatch, provisionFirstTaskFixtures, @@ -336,6 +337,7 @@ export async function runFirstTaskFlow(input: { const agent = await api.get(`/api/agents/${fixtures.agent.id}`); e.configuredModel = agent.adapterConfig?.model ?? null; e.runtimeSettings = { + completionDeliveryProbe: execution.suite.id === "completion-updates", onboardingRuntime: fixtures.onboardingRuntime, adapterType: agent.adapterType, adapterConfig: agent.adapterConfig, @@ -590,6 +592,14 @@ export async function runFirstTaskFlow(input: { scenario.id !== "interview-plan-accept", ); } + if (execution.suite.id === "completion-updates") { + const children = (await api.get(tasksPath)).filter(t => t.parentId === issue.id); + expect(children).toHaveLength(1); + const completion = await observeCompletionUpdate({ ...input, sourceId: issue.id, workerId: children[0]!.id, + marker: scenario.marker, allRuns }); + e.runtimeSettings!.completionRenderedLinks = completion.renderedLinks ?? []; + await snapshot("finished"); + } e.checks = gradeFirstTask(e); await input.evidence("first-task.json", e); await input.capture( diff --git a/tests/runner-e2e/first-task-scoring.ts b/tests/runner-e2e/first-task-scoring.ts index 56decfa0153..d03dfe70f56 100644 --- a/tests/runner-e2e/first-task-scoring.ts +++ b/tests/runner-e2e/first-task-scoring.ts @@ -3,6 +3,7 @@ import { answerableRuntimeRunIds } from "./runtime-question-readiness.js"; import { sanitizeJson } from "./redaction.js"; import { createHash } from "node:crypto"; import { firstTaskScenario } from "./first-task-cases.js"; +import { completionDelivery } from "./completion-updates.js"; export type Row = { id: string; [key: string]: any }; export interface FirstTaskCheckpoint { id: string; @@ -464,5 +465,12 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { ["task-reply-accept", "task-card-accept"].includes(e.caseId) && last.phase === "finished") { checks.push({ ...gradeNativeSessionContinuity(e.checkpoints.flatMap((checkpoint) => checkpoint.runs), e.onboardingIssueId), evidence: [last.id] }); } + if (e.runtimeSettings?.completionDeliveryProbe && last) { + const worker = last.tasks.find(t => t.parentId === e.onboardingIssueId) ?? {}; + checks.push(...completionDelivery({ sourceId: e.onboardingIssueId, worker, documents: last.documents, + comments: last.comments, runs: last.runs, marker: scenario.marker, + renderedLinks: e.runtimeSettings.completionRenderedLinks as Array<{ commentId: string; href: string }> | undefined, + }).checks.map(check => ({ ...check, evidence: [last.id] }))); + } return checks; }