From 8fb291310d665ca7524715861269bd5e0047575c Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 18 Sep 2026 15:22:41 -0500 Subject: [PATCH] Exercise installed legacy skills and grade real continuation outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Initialize the operational skill for legacy continuation fixtures, verify task attachment bytes, allow Claude’s optional custom-answer field, and distinguish unstarted dependency-blocked wakes from failed provider executions. Co-Authored-By: Paperclip --- tests/runner-e2e/continuation-fixtures.test.ts | 13 +++++++++++++ tests/runner-e2e/continuation-fixtures.ts | 10 ++++++++++ tests/runner-e2e/continuation-flow.ts | 14 +++++++++----- tests/runner-e2e/continuation-scoring.ts | 7 ++++++- tests/runner-e2e/continuation.test.ts | 18 ++++++++++++++++++ tests/runner-e2e/first-task-flow.ts | 3 ++- tests/runner-e2e/first-task-scoring.ts | 3 ++- tests/runner-e2e/first-task.test.ts | 11 +++++++++++ tests/runner-e2e/non-execution-wake.test.ts | 9 +++++++++ tests/runner-e2e/non-execution-wake.ts | 5 +++++ tests/runner-e2e/runner.spec.ts | 2 +- .../runtime-question-readiness.test.ts | 11 +++++++++++ tests/runner-e2e/runtime-question-readiness.ts | 7 +++++++ 13 files changed, 104 insertions(+), 9 deletions(-) create mode 100644 tests/runner-e2e/continuation-fixtures.test.ts create mode 100644 tests/runner-e2e/continuation-fixtures.ts create mode 100644 tests/runner-e2e/non-execution-wake.test.ts create mode 100644 tests/runner-e2e/non-execution-wake.ts diff --git a/tests/runner-e2e/continuation-fixtures.test.ts b/tests/runner-e2e/continuation-fixtures.test.ts new file mode 100644 index 00000000000..70b875589fe --- /dev/null +++ b/tests/runner-e2e/continuation-fixtures.test.ts @@ -0,0 +1,13 @@ +import { expect, it, vi } from "vitest"; +import type { RunnerApi } from "./api.js"; +import { prepareLegacyContinuationSkill } from "./continuation-fixtures.js"; +it("initializes the production library and assigns the operational skill before execution", async () => { + const post = vi.fn(); + const get = vi.fn(async () => [{ key: "paperclipai/paperclip/paperclip" }]); + await prepareLegacyContinuationSkill({ get, post } as unknown as RunnerApi, "company", "agent"); + expect(get).toHaveBeenCalledWith("/api/companies/company/skills"); + expect(post).toHaveBeenCalledWith("/api/agents/agent/skills/sync?companyId=company", { desiredSkills: ["paperclipai/paperclip/paperclip"], mode: "add" }); + get.mockResolvedValue([]); + await expect(prepareLegacyContinuationSkill({ get, post } as unknown as RunnerApi, "company", "agent")).rejects.toThrow("missing the bundled"); + expect(post).toHaveBeenCalledTimes(1); +}); diff --git a/tests/runner-e2e/continuation-fixtures.ts b/tests/runner-e2e/continuation-fixtures.ts new file mode 100644 index 00000000000..3faba3d4e4b --- /dev/null +++ b/tests/runner-e2e/continuation-fixtures.ts @@ -0,0 +1,10 @@ +import type { RunnerApi } from "./api.js"; + +/** Bare company creation does not populate its skill library. Match the + * production onboarding setup before evaluating legacy API instructions. */ +export async function prepareLegacyContinuationSkill(api: RunnerApi, companyId: string, agentId: string) { + const key = "paperclipai/paperclip/paperclip"; + const skills = await api.get>(`/api/companies/${companyId}/skills`); + if (!skills.some(skill => skill.key === key)) throw new Error("Continuation fixture is missing the bundled Paperclip operational skill"); + await api.post(`/api/agents/${agentId}/skills/sync?companyId=${companyId}`, { desiredSkills: [key], mode: "add" }); +} diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index e2ce6aba998..97f325367a2 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -1,4 +1,6 @@ -import { answerableRuntimeRunIds } from "./runtime-question-readiness.js"; +import { prepareLegacyContinuationSkill } from "./continuation-fixtures.js"; +import { captureFirstTaskAttachments } from "./first-task-attachments.js"; +import { answerableRuntimeRunIds, isSingleClaudeQuestion } from "./runtime-question-readiness.js"; import { expect, type Page } from "@playwright/test"; import path from "node:path"; import { continuationInitialReady } from "./continuation-readiness.js"; @@ -30,6 +32,7 @@ export async function runContinuationFlow(input: { fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; + secrets: readonly string[]; workspacePath: string; deadlineAt: number; restart(): Promise; @@ -116,7 +119,7 @@ export async function runContinuationFlow(input: { api.get(`/api/issues/${issue!.id}/documents`), api.get(`/api/issues/${issue!.id}/comments?order=asc`), api.get(`/api/issues/${issue!.id}/interactions`), - api.get(`/api/issues/${issue!.id}/attachments`), + captureFirstTaskAttachments(api, [{ id: issue!.id }], input.secrets), ]); const documents = await Promise.all( summaries.map((d) => @@ -159,9 +162,9 @@ export async function runContinuationFlow(input: { ); expect(questions, "one real question must be shown").toHaveLength(1); const set = chatQuestionPresentation(questions[0].payload); - expect(set.questions, "ask only the requested next question").toHaveLength( - 1, - ); + if (scenario.id === "provider-question-bridge") { + expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true); + } else expect(set.questions, "ask only the requested next question").toHaveLength(1); const before = new Set(runs.map((r) => r.id)); if (choice) { expect(set.questions[0].answerMode, "choices must use radio controls").toBe("single_select"); @@ -198,6 +201,7 @@ export async function runContinuationFlow(input: { expect(c.issue.status, "waiting is not complete").not.toBe("done"); } try { + if (execution.profile.generation === "legacy") await prepareLegacyContinuationSkill(api, fixtures.company.id, fixtures.agent.id); await api.patch("/api/instance/settings/experimental", { enableClassicTaskInterface: false, }); diff --git a/tests/runner-e2e/continuation-scoring.ts b/tests/runner-e2e/continuation-scoring.ts index b6402a84ee2..4e58da12443 100644 --- a/tests/runner-e2e/continuation-scoring.ts +++ b/tests/runner-e2e/continuation-scoring.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { gradeQuestionDocumentation } from "./question-documentation-scoring.js"; import type { ContinuationCase } from "./continuation-cases.js"; export interface ContinuationCheckpoint { @@ -55,7 +56,11 @@ export function gradeContinuation(input: { "Record the settled clarification/revision before sending explicit approval.", ); } - const outputs = final?.documents.filter((d) => d.key !== "plan") ?? []; + const verifiedAttachments = (final?.attachments as Array> ?? []).filter(a => + a.contentVerified === true && typeof a.body === "string" && + createHash("sha256").update(a.body).digest("hex") === a.contentSha256); + const outputs = [...(final?.documents.filter((d) => d.key !== "plan") ?? []), + ...verifiedAttachments.map(a => ({ body: a.body as string, latestRevisionId: a.sha256 as string }))]; const output = outputs.length === 1 ? outputs[0] : undefined; check( "updated-output", diff --git a/tests/runner-e2e/continuation.test.ts b/tests/runner-e2e/continuation.test.ts index 4441477ae1e..6987c600a69 100644 --- a/tests/runner-e2e/continuation.test.ts +++ b/tests/runner-e2e/continuation.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { mkdtemp, mkdir, writeFile, readFile, realpath, symlink, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -204,3 +205,20 @@ it("requires a real provider question answered within the same run", () => { expect(failures(r)).toContain("native-question-round-trip"); } }); + +it("grades verified task attachment bytes and rejects metadata-only, tampered, or duplicate output", () => { + const r = recording("answer-updates-scope"); + const final = r.checkpoints.at(-1)!; + const body = final.documents[0].body; + const hash = createHash("sha256").update(body).digest("hex"); + final.documents = []; + const attachment = { id: "file", contentVerified: true, body, sha256: hash, contentSha256: hash }; + final.attachments = [attachment]; + expect(failures(r)).toEqual([]); + final.attachments = [{ ...attachment, contentVerified: false }]; + expect(failures(r)).toContain("updated-output"); + final.attachments = [{ ...attachment, body: body + "tampered" }]; + expect(failures(r)).toContain("updated-output"); + final.attachments = [attachment, { ...attachment, id: "duplicate" }]; + expect(failures(r)).toContain("updated-output"); +}); diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.ts index 99c0dd26de1..705e105d0e8 100644 --- a/tests/runner-e2e/first-task-flow.ts +++ b/tests/runner-e2e/first-task-flow.ts @@ -1,3 +1,4 @@ +import { isBlockedUnstartedWake } from "./non-execution-wake.js"; import { answerableRuntimeRunIds } from "./runtime-question-readiness.js"; import { captureFirstTaskAttachments } from "./first-task-attachments.js"; import { waitForFirstTaskReply } from "./first-task-replies.js"; @@ -272,7 +273,7 @@ export async function runFirstTaskFlow(input: { }), reject: ({ runs }) => { const bad = runs.find((r) => - ["failed", "timed_out", "cancelled"].includes(r.status), + ["failed", "timed_out", "cancelled"].includes(r.status) && !isBlockedUnstartedWake(r), ); if (bad) return `run status ${bad.status}: ${bad.errorCode ?? ""} ${bad.error ?? ""}`; diff --git a/tests/runner-e2e/first-task-scoring.ts b/tests/runner-e2e/first-task-scoring.ts index 7b673b5493c..56decfa0153 100644 --- a/tests/runner-e2e/first-task-scoring.ts +++ b/tests/runner-e2e/first-task-scoring.ts @@ -1,3 +1,4 @@ +import { isBlockedUnstartedWake } from "./non-execution-wake.js"; import { answerableRuntimeRunIds } from "./runtime-question-readiness.js"; import { sanitizeJson } from "./redaction.js"; import { createHash } from "node:crypto"; @@ -454,7 +455,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { ); add( "provider-runs-succeeded", - last.runs.length > 0 && last.runs.every((r) => r.status === "succeeded" || + last.runs.length > 0 && last.runs.every((r) => r.status === "succeeded" || isBlockedUnstartedWake(r) || (scenario.firstResponseOnly && r.status === "running" && answerableRuntimeRunIds(last.interactions).has(r.id))), "Provider runs succeeded, or a first-response run is paused on its recorded answerable native question", [last.id], diff --git a/tests/runner-e2e/first-task.test.ts b/tests/runner-e2e/first-task.test.ts index 2599c630a14..4c41455676a 100644 --- a/tests/runner-e2e/first-task.test.ts +++ b/tests/runner-e2e/first-task.test.ts @@ -1247,3 +1247,14 @@ it("allows first-response native question waits but never treats unfinished jour last.runs[0].status = "failed"; expect(providerPassed()).toBe(false); }); + +it("retains suppressed unstarted wakes without failing successful execution", () => { + const e = recording(); + const last = e.checkpoints.at(-1)!; + const wake = { id: "blocked-wake", status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null as string | null }; + last.runs.push(wake); + const passed = () => gradeFirstTask(e).find(c => c.id === "provider-runs-succeeded")?.passed; + expect(passed()).toBe(true); + wake.startedAt = "2026-09-18"; + expect(passed()).toBe(false); +}); diff --git a/tests/runner-e2e/non-execution-wake.test.ts b/tests/runner-e2e/non-execution-wake.test.ts new file mode 100644 index 00000000000..c17670d6707 --- /dev/null +++ b/tests/runner-e2e/non-execution-wake.test.ts @@ -0,0 +1,9 @@ +import { expect, it } from "vitest"; +import { isBlockedUnstartedWake } from "./non-execution-wake.js"; +it("recognizes only explicitly suppressed wakes that never started execution", () => { + const run = { status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null }; + expect(isBlockedUnstartedWake(run)).toBe(true); + for (const patch of [{ startedAt: "2026-09-18" }, { startedAt: undefined }, { errorCode: "user_cancelled" }, { status: "failed" }]) { + expect(isBlockedUnstartedWake({ ...run, ...patch })).toBe(false); + } +}); diff --git a/tests/runner-e2e/non-execution-wake.ts b/tests/runner-e2e/non-execution-wake.ts new file mode 100644 index 00000000000..6065a5ce728 --- /dev/null +++ b/tests/runner-e2e/non-execution-wake.ts @@ -0,0 +1,5 @@ +/** A queued parent wake can be suppressed while its child still runs. This is + * not a provider failure: admission never started. Keep it in the evidence. */ +export function isBlockedUnstartedWake(run: Record) { + return run.status === "cancelled" && run.errorCode === "issue_dependencies_blocked" && run.startedAt === null; +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index f6b23441b48..19ff9d9cb59 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -791,7 +791,7 @@ for (const execution of executions) { if (execution.task.flow === "continuation") { const continuation = await runContinuationFlow({ - page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs - 60_000, + page, api, fixtures, execution, nonce, secrets, workspacePath, deadlineAt: startedAtMs + deadlineMs - 60_000, restart: () => restartIsolatedPaperclipServer({ api, requestId: `continuation-${nonce}`, deadlineAt: startedAtMs + deadlineMs }), observe: (currentIssue, currentRuns, checks) => { issue = currentIssue; selectedRuns = currentRuns; diff --git a/tests/runner-e2e/runtime-question-readiness.test.ts b/tests/runner-e2e/runtime-question-readiness.test.ts index 2c2b96e2bb4..a25b78d7b19 100644 --- a/tests/runner-e2e/runtime-question-readiness.test.ts +++ b/tests/runner-e2e/runtime-question-readiness.test.ts @@ -5,3 +5,14 @@ it("recognizes only a pending native question as an answerable active turn", () expect([...answerableRuntimeRunIds([native])]).toEqual(["live"]); expect([...answerableRuntimeRunIds([{ ...native, status: "answered" }, { ...native, payload: {} }])]).toEqual([]); }); + +import { isSingleClaudeQuestion } from "./runtime-question-readiness.js"; +it("allows Claude's optional Other companion but rejects another substantive question", () => { + const choice = { id: "choice", answerMode: "single_select" }; + const other = { id: "field-2-question_0_custom-hash", answerMode: "text", required: false, header: "Other" }; + expect(isSingleClaudeQuestion([choice])).toBe(true); + expect(isSingleClaudeQuestion([choice, other])).toBe(true); + expect(isSingleClaudeQuestion([choice, { ...other, required: true }])).toBe(false); + expect(isSingleClaudeQuestion([choice, { ...other, id: "organization", header: "Organization" }])).toBe(false); + expect(isSingleClaudeQuestion([choice, other, other])).toBe(false); +}); diff --git a/tests/runner-e2e/runtime-question-readiness.ts b/tests/runner-e2e/runtime-question-readiness.ts index e70d9d8823e..b26cfa86a77 100644 --- a/tests/runner-e2e/runtime-question-readiness.ts +++ b/tests/runner-e2e/runtime-question-readiness.ts @@ -5,3 +5,10 @@ export function answerableRuntimeRunIds(interactions: ReadonlyArray i.sourceRunId)); } + +/** Claude's one question includes the ACP adapter's optional custom-answer + * companion field. It is not a second user question. */ +export function isSingleClaudeQuestion(questions: ReadonlyArray>) { + return questions.length >= 1 && questions.length <= 2 && questions[0]?.answerMode === "single_select" && + questions.slice(1).every(q => q.answerMode === "text" && q.required === false && q.header === "Other" && /_custom-/.test(q.id)); +}