diff --git a/doc/evals.md b/doc/evals.md index 06037daea6e..61e0d385cf2 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -391,6 +391,11 @@ The 26 native `first-task` cells exercise onboarding before native selection becomes the UI default. Live results and semantic answer reviews must accompany any qualification claim; catalog presence alone is not a pass. +The explicit-only [native question/resume qualification](../tests/runner-e2e/README.md#task-continuation) +separates a completed two-answer user journey from semantic-tool documentation +qualification. It verifies the exact provider-pause or semantic-response-wake +binding for each answer and preserves prior grades when the definition changes. + ## Lifecycle behavior baseline The credential-free [lifecycle baseline](../tests/lifecycle-baseline/README.md) diff --git a/server/src/__tests__/idle-admission.test.ts b/server/src/__tests__/idle-admission.test.ts index 81f268a1ffa..d9b7cd269fc 100644 --- a/server/src/__tests__/idle-admission.test.ts +++ b/server/src/__tests__/idle-admission.test.ts @@ -49,6 +49,36 @@ describe("idle admission", () => { expect(idleWorkSnapshot().active).toBe(0); }); + it("tracks real Vite middleware without confusing Connect mount paths with Express routes", async () => { + const { createServer } = await import("vite"); + const vite = await createServer({ + configFile: false, + appType: "custom", + server: { middlewareMode: true, hmr: false, watch: null }, + optimizeDeps: { noDiscovery: true, include: [] }, + }); + const server = app(), done = deferred(); + vite.middlewares.use("/vite-work", async (_req, res) => { + res.statusCode = 202; + res.end(); + await done.promise; + }); + server.use(vite.middlewares); + try { + expect(() => trackIdleRequestHandlers(server)).not.toThrow(); + await request(server).get("/vite-work").expect(202); + expect(idleWorkSnapshot().active).toBe(1); + startTaskDrain({ purpose: "idle", ttlMs: 60_000 }); + await request(server).get("/vite-work").expect(503); + done.resolve(); + await new Promise(resolve => setImmediate(resolve)); + expect(idleWorkSnapshot().active).toBe(0); + } finally { + done.resolve(); + await vite.close(); + } + }); + it("does not mistake a client disconnect for completed accepted work", async () => { const server = app(); const entered = deferred(), done = deferred(), closed = deferred(); diff --git a/server/src/middleware/idle-admission.ts b/server/src/middleware/idle-admission.ts index c9de5f03eca..9540d704dd0 100644 --- a/server/src/middleware/idle-admission.ts +++ b/server/src/middleware/idle-admission.ts @@ -53,14 +53,16 @@ export const idleAdmissionMiddleware: RequestHandler = (req, res, next) => { // preserving router objects and error-handler arity. Waiting only for finish // loses async work after res.json(), including the disconnected-client case. // Keep this adapter isolated and exercise it against real Express in tests. -type Layer = { handle: Function & { stack?: Layer[] }; route?: { stack: Layer[] } }; +type Layer = { handle: Function & { stack?: Layer[] }; route?: { stack: Layer[] } | string }; export function trackIdleRequestHandlers(app: Application): void { const seen = new Set(); const visit = (stack: Layer[], route = false) => { for (const layer of stack) { if (seen.has(layer)) continue; seen.add(layer); - if (layer.route) { visit(layer.route.stack, true); continue; } + // Connect/Vite layers use a mount-path string for route; only Express + // Route objects contain a nested handler stack. + if (layer.route && typeof layer.route !== "string") { visit(layer.route.stack, true); continue; } if (layer.handle.stack) { visit(layer.handle.stack); continue; } const original = layer.handle; if (original === idleAdmissionMiddleware) continue; diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index fd015d0d628..d24e660c24f 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1271,6 +1271,30 @@ contract tests separately verify that the advertised tool carries the documentat for fresh and resumed native executions. This tests the current documentation placement; it is not a statistical comparison with the former prompt arrangement. +The separate explicit-only `question-resume` suite uses the same neutral two-question +request on native Codex and ACPX Claude. It qualifies the user journey through +either the provider's built-in question or Paperclip's semantic question tool. +It does not replace or regrade `question-tool-documentation`. + +The oracle classifies each wait from its exact task/agent/run identity. Provider +questions require the server-generated request key and retain the paused run. +Semantic questions require an applied `request_human_input` receipt and one new +response wake bound to that interaction and source run. Only a verified provider +choice may carry the adapter's optional Other field. A second substantive question +is still rejected; the subsequent reference question must be text-only. Both real +board answers must remain bound to unchanged forms, and one saved document must +include both answers. Existing lifecycle, ownership, premature-output, terminal +state and no-extra-task checks apply. Three checkpoints are required even when +both answers resume one run; missing native inputs or unexplained runs fail. + +Select `question-resume.runner-acpx-claude.local.question-answer-resume` for the +bounded Claude qualification. The suite is excluded from `--all`; it allows one +attempt, one to three actual runs, a ten-minute cell deadline and verified +1,000-cent company/agent hard stops. The historical failure where the optional +Other field stopped the driver before any answer remains a FAIL. A new behavior +pass does not establish semantic-tool selection, documentation placement, default +hiring behavior or general reliability. + Continuation screenshots wait for the correct task heading and fully revealed conversation before capture. A loading screen or wrong task fails capture. Browser-only regressions exercise delayed rendering without provider calls: diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 6b386d6e182..6e049a4841c 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -146,10 +146,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, + 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, ]); - expect(validateRunnerCatalog()).toHaveLength(698); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(698); + expect(validateRunnerCatalog()).toHaveLength(700); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(700); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 0d2b996f939..bae30990523 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -12,7 +12,7 @@ import { apiResponseReadingTask } from "./api-response-reading.js"; import { taskTitleTasks, taskTitleDefinitionDigest, TASK_TITLE_BUDGET_CENTS } from "./task-titles.js"; import { blockerTasks, blockerProfile } from "./blocker-cases.js"; import { accountingTasks } from "./accounting-cases.js"; -import { continuationTasks } from "./continuation-cases.js"; +import { continuationTasks, questionResumeTask } from "./continuation-cases.js"; import { contextIntegrityTasks, paperclipDocumentTask } from "./context-integrity-cases.js"; import { productionDefaultHireProfile, stockHarnessSourceDigest, stockHarnessSkillSources } from "./stock-harness.js"; import { lifecycleLiveTasks, lifecycleLiveDefinitionDigest } from "./lifecycle-live-cases.js"; @@ -1262,6 +1262,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ excludedExecutionIds: ["neutral", "challenge"].map(variant => `lifecycle-baseline.runner-codex.local.lifecycle-repair-${variant}`), definitionMetadata: { version: 5, clarificationGrading: "wrapped-information-fields-list", narrativeDigest: lifecycleLiveDefinitionDigest, grading: "durable-state-and-attributed-narrative", scheduling: "explicit-only" }, }, + { + id: "question-resume", label: "Native question and answer continuation", manualOnly: true, + description: "Two real UI answers, exact provider-pause or semantic-wake identity, and one saved document.", + groups: ["native", "local"], environments: [localEnvironment], + profiles: runnerProfiles.filter(p => ["runner-codex", "runner-acpx-claude"].includes(p.id)).map(productionStoryProfile), + tasks: [questionResumeTask], expectedMatrixSize: 2, + definitionMetadata: { version: 1, grading: "bound-question-path-and-two-durable-answers", instructions: "unchanged-production", + scheduling: "explicit-only", automaticRetryPolicy: "single_attempt", companyAndAgentBudgetCents: 1_000, + providerRuns: "1-3; each semantic answer starts one exactly bound wake; provider answers retain their paused run" }, + }, { id: "continuation", label: "Task continuation", description: "Human direction, approval boundaries, untrusted evidence, and completed actions across turns.", @@ -1668,6 +1678,7 @@ export function validateRunnerCatalog(): MatrixExecution[] { ...accountingTasks, ...lifecycleLiveTasks, ...continuationTasks, + questionResumeTask, ...everydayTasks, ...taskTitleTasks, ...runnerTasks, diff --git a/tests/runner-e2e/continuation-cases.ts b/tests/runner-e2e/continuation-cases.ts index 65810ad4fb7..781d44e394c 100644 --- a/tests/runner-e2e/continuation-cases.ts +++ b/tests/runner-e2e/continuation-cases.ts @@ -10,9 +10,9 @@ export const CONTINUATION_CASES = [ "question-tool-documentation", "provider-question-bridge", ] as const; -export type ContinuationCase = (typeof CONTINUATION_CASES)[number]; +export type ContinuationCase = (typeof CONTINUATION_CASES)[number] | "question-answer-resume"; export function continuationScenario(id: string, nonce: string) { - if (!CONTINUATION_CASES.includes(id as ContinuationCase)) + if (id !== "question-answer-resume" && !CONTINUATION_CASES.includes(id as (typeof CONTINUATION_CASES)[number])) throw new Error(`Unknown continuation case: ${id}`); const marker = `AMBER${nonce.replace(/[^a-z0-9]/gi, "")}`; const old = `COBALT${nonce.replace(/[^a-z0-9]/gi, "")}`; @@ -21,9 +21,11 @@ export function continuationScenario(id: string, nonce: string) { const childTitle = `Follow-up ${marker}`; const document = "Save the final note as a task document. No other deliverables or child tasks are needed."; + const questionPrompt = "Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables."; const prompts: Record = { + "question-answer-resume": questionPrompt, "provider-question-bridge": `Use your built-in AskUserQuestion tool (not Paperclip's request_human_input) to ask which reference to include, with two choices: ${marker} and ${old}. Wait for my real answer, then save a one-sentence welcome note including only my selected reference as a task document and finish. No other tasks or deliverables are needed.`, - "question-tool-documentation": `Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables.`, + "question-tool-documentation": questionPrompt, "answer-updates-scope": `I need a one-sentence welcome note containing ${old}. Before writing it, ask me one open-ended structured question about any changes I want. Then apply my answer and finish. ${document}`, "clarification-not-approval": `I need a one-sentence welcome note. First ask me one open-ended structured question for the word to include. After my answer, propose your approach and wait for my explicit approval before writing the note. ${document}`, "revision-preserves-approval": `Propose an approach for a one-sentence welcome note containing ${old}, and save that approach as the plan document. Wait for my explicit approval before writing the note. ${document}`, @@ -85,3 +87,12 @@ export function continuationScreenshotFile( ? "final-state.png" : `question-continuation-${phase}.png`; } + +/** Separate behavior contract; leave the semantic documentation case unchanged. */ +export const questionResumeTask: RunnerTaskFixture = { + ...continuationTasks.find(t => t.id === "question-tool-documentation")!, + id: "question-answer-resume", label: "Two answered questions across native paths", + expectedRunCount: 3, minimumExpectedRunCount: 1, + buildTitle: nonce => `Question resume ${nonce}`, + buildPrompt: nonce => continuationScenario("question-answer-resume", nonce).prompt, +}; diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index 6381d03b5c4..8c0ad9ea222 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -1,3 +1,4 @@ +import { validResumeQuestionForm } from "./question-resume-scoring.js"; import { gradeLifecycleBaseline, type LifecycleCheckpoint } from "./lifecycle-baseline.js"; import { lifecycleLiveCase, lifecycleLiveContinuation, gradeLifecycleNarrative } from "./lifecycle-live-cases.js"; import { prepareLegacyContinuationSkill, prepareContinuationBudget } from "./continuation-fixtures.js"; @@ -106,7 +107,7 @@ export async function runContinuationFlow(input: { return ready; }, reject: (state) => - state.runs.length > 12 + state.runs.length > (scenario.id === "question-answer-resume" ? 3 : 12) ? "Bounded continuation run count exceeded" : state.runs.some((r) => ["failed", "timed_out", "cancelled"].includes(r.status), @@ -185,7 +186,9 @@ export async function runContinuationFlow(input: { ); expect(questions, "one real question must be shown").toHaveLength(1); const set = chatQuestionPresentation(questions[0].payload); - if (scenario.id === "provider-question-bridge") { + if (scenario.id === "question-answer-resume") { + expect(validResumeQuestionForm(checkpoints.at(-1)!, questions[0], !!choice), "a verified native question path and the requested form").toBe(true); + } else if (scenario.id === "provider-question-bridge") { expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true); } else expect(set.questions, "ask only the requested next question").toHaveLength(1); const before = new Set(runs.map((r) => r.id)); @@ -267,7 +270,7 @@ export async function runContinuationFlow(input: { await input.restart(); await open(); } - if (scenario.id === "question-tool-documentation") { + if (["question-tool-documentation", "question-answer-resume"].includes(scenario.id)) { await answer("Afternoon"); await snapshot("answered"); assertWaiting(); diff --git a/tests/runner-e2e/continuation-scoring.ts b/tests/runner-e2e/continuation-scoring.ts index ada89b09167..42e79f8ed0b 100644 --- a/tests/runner-e2e/continuation-scoring.ts +++ b/tests/runner-e2e/continuation-scoring.ts @@ -1,3 +1,4 @@ +import { gradeQuestionResume } from "./question-resume-scoring.js"; import { createHash } from "node:crypto"; import { gradeQuestionDocumentation } from "./question-documentation-scoring.js"; import type { ContinuationCase } from "./continuation-cases.js"; @@ -49,7 +50,7 @@ export function gradeContinuation(input: { const before = input.checkpoints.filter((c) => c.phase !== "final"); check( "recorded-continuation", - before.length > 0 && !!final && final.runs.length >= (input.id === "provider-question-bridge" ? 1 : 2), + before.length > 0 && !!final && final.runs.length >= (["provider-question-bridge", "question-answer-resume"].includes(input.id) ? 1 : 2), "Initial and final turns must both be recorded.", ); for (const c of before) { @@ -135,5 +136,6 @@ export function gradeContinuation(input: { "A real provider-native card must be answered and resume the same run to completion."); } if (input.id === "question-tool-documentation") checks.push(...gradeQuestionDocumentation(input.checkpoints, input.marker)); + if (input.id === "question-answer-resume") checks.push(...gradeQuestionResume(input.checkpoints, input.marker)); return checks; } diff --git a/tests/runner-e2e/question-resume-scoring.ts b/tests/runner-e2e/question-resume-scoring.ts new file mode 100644 index 00000000000..4f494bdae04 --- /dev/null +++ b/tests/runner-e2e/question-resume-scoring.ts @@ -0,0 +1,131 @@ +import { isDeepStrictEqual } from "node:util"; +import type { ContinuationCheckpoint } from "./continuation-scoring.js"; +import { isSingleClaudeQuestion } from "./runtime-question-readiness.js"; + +type Row = Record; +export type QuestionPath = "provider" | "semantic"; +const nonempty = (v: unknown): v is string => typeof v === "string" && v.trim().length > 0; +const questions = (c?: ContinuationCheckpoint): Row[] => + (c?.interactions ?? []).filter((i: any) => i.kind === "ask_user_questions") as Row[]; +export const resumeQuestionFields = (i?: Row): Row[] => + Array.isArray(i?.payload?.questionSet?.questions) ? i.payload.questionSet.questions : []; + +/** Classify by server-owned identities and the applied semantic receipt. A + * missing receipt is unknown, never inferred from form shape or run count. */ +export function questionPath(checkpoint: ContinuationCheckpoint | undefined, i?: Row): QuestionPath | null { + if (!checkpoint || !i || !nonempty(i.id) || !nonempty(i.sourceRunId) || !nonempty(i.idempotencyKey)) return null; + const matches = checkpoint.runs.filter(r => r.id === i.sourceRunId) as Row[]; + const run = matches.length === 1 ? matches[0] : undefined; + if (!run || run.runtimeMode !== "native" || !nonempty(run.companyId) || + run.companyId !== (checkpoint.issue as Row).companyId || + i.companyId !== run.companyId || i.issueId !== checkpoint.issue.id || run.issueId !== i.issueId || + !nonempty(checkpoint.issue.assigneeAgentId) || i.createdByAgentId !== checkpoint.issue.assigneeAgentId || + run.agentId !== checkpoint.issue.assigneeAgentId) return null; + const request = i.payload?.runtimeRequestId; + if (request !== undefined) { + return nonempty(request) && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$/.test(request) && + i.idempotencyKey === `paperclip-runner-question:${run.id}:${request}` && + i.continuationPolicy === "none" && run.status === "running" ? "provider" : null; + } + const receipt = run.resultJson?.semanticToolReceipts?.[i.idempotencyKey]; + const created = receipt?.result?.interaction; + return run.status === "succeeded" && i.continuationPolicy === "wake_assignee" && + receipt?.operationId === "request_human_input" && receipt.result?.disposition === "applied" && + created?.id === i.id && created?.sourceRunId === run.id && created?.issueId === i.issueId && + created?.companyId === i.companyId && created?.kind === "ask_user_questions" && + isDeepStrictEqual(created?.payload?.questionSet, i.payload?.questionSet) ? "semantic" : null; +} + +export function validResumeQuestionForm(checkpoint: ContinuationCheckpoint, i: Row, choice: boolean): boolean { + const path = questionPath(checkpoint, i); + const fields = resumeQuestionFields(i); + if (!path || fields.some(f => !nonempty(f.id)) || new Set(fields.map(f => f.id)).size !== fields.length) return false; + if (!choice) return fields.length === 1 && fields[0].answerMode === "text" && + !fields[0].options?.length && !fields[0].customAnswer; + return fields[0]?.answerMode === "single_select" && + (fields.length === 1 || (path === "provider" && isSingleClaudeQuestion(fields))); +} + +function sameQuestion(a: Row, b?: Row) { + return !!b && ["id", "kind", "sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey", "continuationPolicy"] + .every(k => a[k] === b[k]) && isDeepStrictEqual(a.payload, b.payload); +} +function answeredByUser(i?: Row) { + return i?.status === "answered" && nonempty(i.resolvedByUserId) && !i.resolvedByAgentId; +} +function resumedRun(before: ContinuationCheckpoint | undefined, after: ContinuationCheckpoint | undefined, i?: Row) { + if (!before || !after || !i) return false; + const path = questionPath(before, i); + const prior = new Set(before.runs.map(r => r.id)); + const next = after.runs.filter(r => !prior.has(r.id)) as Row[]; + if (prior.size !== before.runs.length || new Set(after.runs.map(r => r.id)).size !== after.runs.length || + before.runs.some(r => !after.runs.some(n => n.id === r.id))) return false; + if (path === "provider") return next.length === 0 && + after.runs.some(r => r.id === i.sourceRunId && ["running", "succeeded"].includes(r.status)); + if (path !== "semantic" || next.length !== 1) return false; + const context = next[0].contextSnapshot; + return context?.issueId === i.issueId && context?.sourceRunId === i.sourceRunId && + context?.interactionId === i.id && context?.interactionKind === "ask_user_questions" && + context?.interactionStatus === "answered"; +} + +/** Behavioral qualification supports either valid native path. It does not + * replace the separate documentation-placement/semantic-tool qualification. */ +export function gradeQuestionResume(checkpoints: ContinuationCheckpoint[], marker: string) { + const initial = checkpoints.find(c => c.phase === "initial"); + const middle = checkpoints.find(c => c.phase === "answered"); + const final = checkpoints.find(c => c.phase === "final"); + const first = questions(initial)[0]; + const second = questions(middle).find(i => i.status === "pending"); + const choice = resumeQuestionFields(first)[0]; + const text = resumeQuestionFields(second)[0]; + const options: Row[] = choice?.options ?? []; + const afternoon = options.find(o => /^afternoon\b/i.test(String(o.label).trim())); + const checks = [{ + id: "question-resume.forms", + passed: !!initial && !!middle && questions(initial).length === 1 && first?.status === "pending" && + validResumeQuestionForm(initial, first, true) && options.length >= 2 && + options.every(o => nonempty(o.id)) && new Set(options.map(o => o.id)).size === options.length && + options.some(o => /^morning\b/i.test(String(o.label).trim())) && !!afternoon && + questions(middle).length === 2 && questions(middle).filter(i => i.status === "pending").length === 1 && + nonempty(second?.id) && second?.id !== first?.id && validResumeQuestionForm(middle, second, false), + detail: "One choice plus only a verified provider Other companion, then a distinct text-only question.", + }]; + for (const [before, after, i, field, answer] of [ + [initial, middle, first, choice, "choice"], [middle, final, second, text, "text"], + ] as const) { + const committed = questions(after).filter(q => q.id === i?.id); + const retained = questions(final).filter(q => q.id === i?.id); + const response: Row[] = committed[0]?.result?.answers ?? []; + const primary = response.filter(a => a.questionId === field?.id); + const optionalIds = resumeQuestionFields(i).slice(1).map(f => f.id); + const validAnswer = primary.length === 1 && (answer === "choice" + ? primary[0].optionIds?.length === 1 && primary[0].optionIds[0] === afternoon?.id && !primary[0].otherText + : !primary[0].optionIds?.length && primary[0].otherText === `Use ${marker}.`); + checks.push({ + id: `question-resume.${answer}.answer`, + passed: !!i && committed.length === 1 && retained.length === 1 && + sameQuestion(i, committed[0]) && sameQuestion(i, retained[0]) && answeredByUser(committed[0]) && + answeredByUser(retained[0]) && isDeepStrictEqual(committed[0].result, retained[0].result) && + validAnswer && response.every(a => a.questionId === field?.id || + (optionalIds.includes(a.questionId) && !a.optionIds?.length && !a.otherText)), + detail: "The real board answer must remain bound to the unchanged question, with no fabricated or substituted response.", + }, { + id: `question-resume.${answer}.path`, + passed: resumedRun(before, after, i), + detail: `Observed path: ${questionPath(before, i) ?? "unknown"}; provider answers resume the same run; semantic answers require one exactly bound response wake.`, + }); + } + checks.push({ + id: "question-resume.complete-evidence", + passed: checkpoints.length === 3 && !!initial && !!middle && !!final && initial.runs.length === 1 && + questions(final).length === 2 && final.runs.length >= 1 && final.runs.length <= 3 && + final.runs.every((r: Row) => r.companyId === (final.issue as Row).companyId && + r.issueId === final.issue.id && r.agentId === final.issue.assigneeAgentId && + typeof r.runnerProfileJson?.nativeExecutionInput?.task?.prompt === "string") && + final.documents.length === 1 && !!final.documents[0].latestRevisionId && + final.documents[0].body.includes(`Use ${marker}.`) && /\bafternoon\b/i.test(final.documents[0].body), + detail: "Retain all three checkpoints, every native input and exactly one saved document containing the choice and the complete submitted reference.", + }); + return checks.map(c => ({ ...c, passed: Boolean(c.passed) })); +} diff --git a/tests/runner-e2e/question-resume.test.ts b/tests/runner-e2e/question-resume.test.ts new file mode 100644 index 00000000000..525bce0c199 --- /dev/null +++ b/tests/runner-e2e/question-resume.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import { gradeQuestionResume, questionPath, validResumeQuestionForm, type QuestionPath } from "./question-resume-scoring.js"; +import { gradeContinuation } from "./continuation-scoring.js"; +import { gradeLifecycleBaseline } from "./lifecycle-baseline.js"; +import { continuationScenario } from "./continuation-cases.js"; +import { runnerMatrix } from "./catalog.js"; + +type Row = Record; +function recording(paths: QuestionPath[] = ["provider", "semantic"]) { + const runs: Row[] = []; + const cards: Row[] = []; + const points: any[] = []; + const newRun = (previous?: Row) => ({ id: `run-${runs.length + 1}`, status: "running", runtimeMode: "native", + companyId: "company", issueId: "task", agentId: "agent", resultJson: { semanticToolReceipts: {} }, + runnerProfileJson: { nativeExecutionInput: { task: { prompt: "actual native task input" } } }, + contextSnapshot: previous ? { issueId: "task", sourceRunId: previous.sourceRunId, + interactionId: previous.id, interactionKind: "ask_user_questions", interactionStatus: "answered" } : {}, + }); + runs.push(newRun()); + const snap = (phase: string, documents: Row[] = []) => { + const active = runs.find(r => r.status === "running"); + points.push(structuredClone({ phase, issue: { id: "task", companyId: "company", assigneeAgentId: "agent", + status: phase === "final" ? "done" : active ? "in_progress" : "in_review" }, + lifecycle: { executionRunId: active?.id ?? null, scheduledRetry: null, activeRecoveryAction: null, monitorNextCheckAt: null }, + runs, interactions: cards, children: [], attachments: [], comments: [], documents })); + }; + for (let n = 0; n < 2; n++) { + const run = runs.at(-1)!; + const provider = paths[n] === "provider"; + const request = `request-${n}`; + const idempotencyKey = provider ? `paperclip-runner-question:${run.id}:${request}` : `question-${n}`; + const field = n === 0 ? { id: "field-1-question_0-hash", answerMode: "single_select", options: [ + { id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" }, + ] } : { id: "reference", answerMode: "text" }; + const questions: Row[] = [field]; + if (provider && n === 0) questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false }); + const card: Row = { id: `card-${n}`, kind: "ask_user_questions", status: "pending", + companyId: "company", issueId: "task", createdByAgentId: "agent", sourceRunId: run.id, idempotencyKey, + continuationPolicy: provider ? "none" : "wake_assignee", + payload: { ...(provider ? { runtimeRequestId: request } : {}), questionSet: { questions } } }; + cards.push(card); + if (!provider) { + run.status = "succeeded"; + run.resultJson.semanticToolReceipts[idempotencyKey] = { operationId: "request_human_input", + result: { disposition: "applied", interaction: structuredClone(card) } }; + } + snap(n === 0 ? "initial" : "answered"); + card.status = "answered"; card.resolvedByUserId = "local-board"; + card.result = { answers: [n === 0 ? { questionId: field.id, optionIds: ["pm"] } : + { questionId: field.id, optionIds: [], otherText: "Use AMBERnonce." }] }; + if (!provider) runs.push(newRun(card)); + } + runs.at(-1)!.status = "succeeded"; + snap("final", [{ key: "welcome-note", latestRevisionId: "revision", body: "Welcome to the afternoon meetup. Use AMBERnonce." }]); + return points; +} +const failed = (r: any[]) => gradeQuestionResume(r, "AMBERnonce").filter(c => !c.passed).map(c => c.id); +const question = (r: any[], n = 0) => r[n].interactions[n]; + +describe("native question/resume path qualification", () => { + it("keeps the new journey explicit-only and the original documentation cells intact", () => { + const cells = runnerMatrix.filter(c => c.suite.id === "question-resume"); + expect(cells.map(c => c.profile.id).sort()).toEqual(["runner-acpx-claude", "runner-codex"]); + for (const c of cells) { + expect(c.suite.manualOnly).toBe(true); + expect(c.task).toMatchObject({ expectedRunCount: 3, minimumExpectedRunCount: 1, automaticRetryPolicy: "single_attempt" }); + expect(c.task.buildPrompt("nonce")).toBe(continuationScenario("question-tool-documentation", "nonce").prompt); + } + expect(runnerMatrix.filter(c => c.suite.id === "continuation")).toHaveLength(23); + }); + it.each([ + ["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"], + ] as QuestionPath[][])("accepts exactly bound %s then %s journeys", (a, b) => { + const r = recording([a,b]); + expect(failed(r)).toEqual([]); + expect(gradeContinuation({ ...continuationScenario("question-answer-resume", "nonce"), runtimeMode: "native", checkpoints: r }).filter(c => !c.passed)).toEqual([]); + expect(gradeLifecycleBaseline(r).filter(c => !c.passed)).toEqual([]); + }); + it.each([ + "Welcome to the afternoon meetup, AMBERnonce.", + "Welcome to the afternoon meetup. Use AMBERnonce", + ])("rejects a shortened supplied reference in the saved document: %s", body => { + const r = recording(); + r[2].documents[0].body = body; + expect(failed(r)).toContain("question-resume.complete-evidence"); + }); + it("does not admit an optional Other field on the semantic path", () => { + const r = recording(["semantic", "semantic"]); + const i = question(r); + i.payload.questionSet.questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false }); + expect(validResumeQuestionForm(r[0], i, true)).toBe(false); + }); + it.each(["sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey"])("rejects a provider card with mismatched %s", key => { + const r = recording(); question(r)[key] = "unrelated"; + expect(failed(r)).toContain("question-resume.choice.path"); + }); + it("rejects a consistent foreign-company card/run pair on this task", () => { + const r = recording(); question(r).companyId = "foreign"; r[0].runs[0].companyId = "foreign"; + expect(questionPath(r[0], question(r))).toBeNull(); + }); + it.each(["missing-request", "wrong-request", "terminal-run", "wrong-policy", "legacy-run"])("rejects invalid provider binding: %s", kind => { + const r = recording(); const i = question(r); + if (kind === "missing-request") delete i.payload.runtimeRequestId; + if (kind === "wrong-request") i.payload.runtimeRequestId = "unrelated"; + if (kind === "terminal-run") r[0].runs[0].status = "succeeded"; + if (kind === "wrong-policy") i.continuationPolicy = "wake_assignee"; + if (kind === "legacy-run") r[0].runs[0].runtimeMode = "legacy"; + expect(questionPath(r[0], i)).toBeNull(); + }); + it.each(["missing", "wrong-operation", "rejected", "wrong-interaction", "wrong-source", "wrong-form"])("rejects missing or mismatched semantic receipt: %s", kind => { + const r = recording(["semantic", "semantic"]); const i = question(r); + const receipt = r[0].runs[0].resultJson.semanticToolReceipts[i.idempotencyKey]; + if (kind === "missing") delete r[0].runs[0].resultJson.semanticToolReceipts[i.idempotencyKey]; + if (kind === "wrong-operation") receipt.operationId = "write_document"; + if (kind === "rejected") receipt.result.disposition = "rejected"; + if (kind === "wrong-interaction") receipt.result.interaction.id = "unrelated"; + if (kind === "wrong-source") receipt.result.interaction.sourceRunId = "unrelated"; + if (kind === "wrong-form") receipt.result.interaction.payload.questionSet.questions = []; + expect(questionPath(r[0], i)).toBeNull(); + }); + it.each(["required-other", "extra-question", "text-with-options", "two-early-cards"])("rejects the wrong form: %s", kind => { + const r = recording(); const fields = question(r).payload.questionSet.questions; + if (kind === "required-other") fields[1].required = true; + if (kind === "extra-question") fields.push({ id: "extra", answerMode: "text" }); + if (kind === "text-with-options") question(r,1).payload.questionSet.questions[0].options = [{ id: "other" }]; + if (kind === "two-early-cards") r[0].interactions.push(structuredClone(question(r,1))); + expect(failed(r)).toContain("question-resume.forms"); + }); + it.each(["agent-answer", "wrong-choice", "wrong-text", "duplicate-answer", "replaced-card", "changed-form", "changed-answer", "unanswered"])("rejects corrupt answer evidence: %s", kind => { + const r = recording(); const first = r[1].interactions[0]; const last = r[2].interactions[1]; + if (kind === "agent-answer") { first.resolvedByUserId = null; first.resolvedByAgentId = "agent"; } + if (kind === "wrong-choice") first.result.answers[0].optionIds = ["am"]; + if (kind === "wrong-text") last.result.answers[0].otherText = "wrong"; + if (kind === "duplicate-answer") first.result.answers.push(structuredClone(first.result.answers[0])); + if (kind === "replaced-card") last.id = "replacement"; + if (kind === "changed-form") last.payload.questionSet.questions[0].prompt = "Changed question"; + if (kind === "changed-answer") r[2].interactions[0].result.answers[0].optionIds = ["am"]; + if (kind === "unanswered") last.status = "pending"; + expect(failed(r).some(id => id.endsWith(".answer"))).toBe(true); + }); + it("rejects replacing a paused provider run with a fresh run", () => { + const r = recording(); r[1].runs.push({ ...structuredClone(r[1].runs[0]), id: "replacement" }); + expect(failed(r)).toContain("question-resume.choice.path"); + }); + it.each(["interactionId", "sourceRunId", "issueId", "interactionStatus"])("rejects the wrong semantic response wake %s", key => { + const r = recording(); r[2].runs[1].contextSnapshot[key] = "unrelated"; + expect(failed(r)).toContain("question-resume.text.path"); + }); + it.each(["lost-run", "duplicate-run", "extra-run", "missing-input", "missing-final", "missing-middle", "missing-output", "wrong-output"])("rejects incomplete continuation evidence: %s", kind => { + const r = recording(); + if (kind === "lost-run") r[2].runs.shift(); + if (kind === "duplicate-run") r[2].runs.push(structuredClone(r[2].runs[0])); + if (kind === "extra-run") r[2].runs.push({ ...structuredClone(r[2].runs[0]), id: "extra" }); + if (kind === "missing-input") delete r[2].runs[0].runnerProfileJson; + if (kind === "missing-output") r[2].documents = []; + if (kind === "wrong-output") r[2].documents[0].body = "Welcome AMBERnonce."; + if (kind === "missing-final") r.pop(); + if (kind === "missing-middle") r.splice(1,1); + expect(failed(r).length).toBeGreaterThan(0); + }); +});