Exercise installed legacy skills and grade real continuation outputs

Initialize the operational skill for legacy continuation fixtures, verify task attachment bytes, allow Claude’s optional custom-answer field, and distinguish unstarted dependency-blocked wakes from failed provider executions.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/continuation-fixtures.test.ts

@@ -0,0 +1,13 @@
+import { expect, it, vi } from "vitest";
+import type { RunnerApi } from "./api.js";
+import { prepareLegacyContinuationSkill } from "./continuation-fixtures.js";
+it("initializes the production library and assigns the operational skill before execution", async () => {
+  const post = vi.fn();
+  const get = vi.fn(async () => [{ key: "paperclipai/paperclip/paperclip" }]);
+  await prepareLegacyContinuationSkill({ get, post } as unknown as RunnerApi, "company", "agent");
+  expect(get).toHaveBeenCalledWith("/api/companies/company/skills");
+  expect(post).toHaveBeenCalledWith("/api/agents/agent/skills/sync?companyId=company", { desiredSkills: ["paperclipai/paperclip/paperclip"], mode: "add" });
+  get.mockResolvedValue([]);
+  await expect(prepareLegacyContinuationSkill({ get, post } as unknown as RunnerApi, "company", "agent")).rejects.toThrow("missing the bundled");
+  expect(post).toHaveBeenCalledTimes(1);
+});

tests/runner-e2e/continuation-fixtures.ts

@@ -0,0 +1,10 @@
+import type { RunnerApi } from "./api.js";
+
+/** Bare company creation does not populate its skill library. Match the
+ * production onboarding setup before evaluating legacy API instructions. */
+export async function prepareLegacyContinuationSkill(api: RunnerApi, companyId: string, agentId: string) {
+  const key = "paperclipai/paperclip/paperclip";
+  const skills = await api.get<Array<{ key: string }>>(`/api/companies/${companyId}/skills`);
+  if (!skills.some(skill => skill.key === key)) throw new Error("Continuation fixture is missing the bundled Paperclip operational skill");
+  await api.post(`/api/agents/${agentId}/skills/sync?companyId=${companyId}`, { desiredSkills: [key], mode: "add" });
+}

tests/runner-e2e/continuation-flow.ts

@@ -1,4 +1,6 @@
-import { answerableRuntimeRunIds } from "./runtime-question-readiness.js";
+import { prepareLegacyContinuationSkill } from "./continuation-fixtures.js";
+import { captureFirstTaskAttachments } from "./first-task-attachments.js";
+import { answerableRuntimeRunIds, isSingleClaudeQuestion } from "./runtime-question-readiness.js";
 import { expect, type Page } from "@playwright/test";
 import path from "node:path";
 import { continuationInitialReady } from "./continuation-readiness.js";
@@ -30,6 +32,7 @@ export async function runContinuationFlow(input: {
   fixtures: LiveFixtureValues;
   execution: MatrixExecution;
   nonce: string;
+  secrets: readonly string[];
   workspacePath: string;
   deadlineAt: number;
   restart(): Promise<void>;
@@ -116,7 +119,7 @@ export async function runContinuationFlow(input: {
         api.get<Row[]>(`/api/issues/${issue!.id}/documents`),
         api.get<Row[]>(`/api/issues/${issue!.id}/comments?order=asc`),
         api.get<Row[]>(`/api/issues/${issue!.id}/interactions`),
-        api.get<Row[]>(`/api/issues/${issue!.id}/attachments`),
+        captureFirstTaskAttachments(api, [{ id: issue!.id }], input.secrets),
       ]);
     const documents = await Promise.all(
       summaries.map((d) =>
@@ -159,9 +162,9 @@ export async function runContinuationFlow(input: {
     );
     expect(questions, "one real question must be shown").toHaveLength(1);
     const set = chatQuestionPresentation(questions[0].payload);
-    expect(set.questions, "ask only the requested next question").toHaveLength(
-      1,
-    );
+    if (scenario.id === "provider-question-bridge") {
+      expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true);
+    } else expect(set.questions, "ask only the requested next question").toHaveLength(1);
     const before = new Set(runs.map((r) => r.id));
     if (choice) {
       expect(set.questions[0].answerMode, "choices must use radio controls").toBe("single_select");
@@ -198,6 +201,7 @@ export async function runContinuationFlow(input: {
     expect(c.issue.status, "waiting is not complete").not.toBe("done");
   }
   try {
+    if (execution.profile.generation === "legacy") await prepareLegacyContinuationSkill(api, fixtures.company.id, fixtures.agent.id);
     await api.patch("/api/instance/settings/experimental", {
       enableClassicTaskInterface: false,
     });

tests/runner-e2e/continuation-scoring.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { gradeQuestionDocumentation } from "./question-documentation-scoring.js";
 import type { ContinuationCase } from "./continuation-cases.js";
 export interface ContinuationCheckpoint {
@@ -55,7 +56,11 @@ export function gradeContinuation(input: {
       "Record the settled clarification/revision before sending explicit approval.",
     );
   }
-  const outputs = final?.documents.filter((d) => d.key !== "plan") ?? [];
+  const verifiedAttachments = (final?.attachments as Array<Record<string, any>> ?? []).filter(a =>
+    a.contentVerified === true && typeof a.body === "string" &&
+    createHash("sha256").update(a.body).digest("hex") === a.contentSha256);
+  const outputs = [...(final?.documents.filter((d) => d.key !== "plan") ?? []),
+    ...verifiedAttachments.map(a => ({ body: a.body as string, latestRevisionId: a.sha256 as string }))];
   const output = outputs.length === 1 ? outputs[0] : undefined;
   check(
     "updated-output",

tests/runner-e2e/continuation.test.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { mkdtemp, mkdir, writeFile, readFile, realpath, symlink, rm } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
@@ -204,3 +205,20 @@ it("requires a real provider question answered within the same run", () => {
     expect(failures(r)).toContain("native-question-round-trip");
   }
 });
+
+it("grades verified task attachment bytes and rejects metadata-only, tampered, or duplicate output", () => {
+  const r = recording("answer-updates-scope");
+  const final = r.checkpoints.at(-1)!;
+  const body = final.documents[0].body;
+  const hash = createHash("sha256").update(body).digest("hex");
+  final.documents = [];
+  const attachment = { id: "file", contentVerified: true, body, sha256: hash, contentSha256: hash };
+  final.attachments = [attachment];
+  expect(failures(r)).toEqual([]);
+  final.attachments = [{ ...attachment, contentVerified: false }];
+  expect(failures(r)).toContain("updated-output");
+  final.attachments = [{ ...attachment, body: body + "tampered" }];
+  expect(failures(r)).toContain("updated-output");
+  final.attachments = [attachment, { ...attachment, id: "duplicate" }];
+  expect(failures(r)).toContain("updated-output");
+});

tests/runner-e2e/first-task-flow.ts

@@ -1,3 +1,4 @@
+import { isBlockedUnstartedWake } from "./non-execution-wake.js";
 import { answerableRuntimeRunIds } from "./runtime-question-readiness.js";
 import { captureFirstTaskAttachments } from "./first-task-attachments.js";
 import { waitForFirstTaskReply } from "./first-task-replies.js";
@@ -272,7 +273,7 @@ export async function runFirstTaskFlow(input: {
       }),
       reject: ({ runs }) => {
         const bad = runs.find((r) =>
-          ["failed", "timed_out", "cancelled"].includes(r.status),
+          ["failed", "timed_out", "cancelled"].includes(r.status) && !isBlockedUnstartedWake(r),
         );
         if (bad)
           return `run status ${bad.status}: ${bad.errorCode ?? ""} ${bad.error ?? ""}`;

tests/runner-e2e/first-task-scoring.ts

@@ -1,3 +1,4 @@
+import { isBlockedUnstartedWake } from "./non-execution-wake.js";
 import { answerableRuntimeRunIds } from "./runtime-question-readiness.js";
 import { sanitizeJson } from "./redaction.js";
 import { createHash } from "node:crypto";
@@ -454,7 +455,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] {
     );
   add(
     "provider-runs-succeeded",
-    last.runs.length > 0 && last.runs.every((r) => r.status === "succeeded" ||
+    last.runs.length > 0 && last.runs.every((r) => r.status === "succeeded" || isBlockedUnstartedWake(r) ||
       (scenario.firstResponseOnly && r.status === "running" && answerableRuntimeRunIds(last.interactions).has(r.id))),
     "Provider runs succeeded, or a first-response run is paused on its recorded answerable native question",
     [last.id],

tests/runner-e2e/first-task.test.ts

@@ -1247,3 +1247,14 @@ it("allows first-response native question waits but never treats unfinished jour
   last.runs[0].status = "failed";
   expect(providerPassed()).toBe(false);
 });
+
+it("retains suppressed unstarted wakes without failing successful execution", () => {
+  const e = recording();
+  const last = e.checkpoints.at(-1)!;
+  const wake = { id: "blocked-wake", status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null as string | null };
+  last.runs.push(wake);
+  const passed = () => gradeFirstTask(e).find(c => c.id === "provider-runs-succeeded")?.passed;
+  expect(passed()).toBe(true);
+  wake.startedAt = "2026-09-18";
+  expect(passed()).toBe(false);
+});

tests/runner-e2e/non-execution-wake.test.ts

@@ -0,0 +1,9 @@
+import { expect, it } from "vitest";
+import { isBlockedUnstartedWake } from "./non-execution-wake.js";
+it("recognizes only explicitly suppressed wakes that never started execution", () => {
+  const run = { status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null };
+  expect(isBlockedUnstartedWake(run)).toBe(true);
+  for (const patch of [{ startedAt: "2026-09-18" }, { startedAt: undefined }, { errorCode: "user_cancelled" }, { status: "failed" }]) {
+    expect(isBlockedUnstartedWake({ ...run, ...patch })).toBe(false);
+  }
+});

tests/runner-e2e/non-execution-wake.ts

@@ -0,0 +1,5 @@
+/** A queued parent wake can be suppressed while its child still runs. This is
+ * not a provider failure: admission never started. Keep it in the evidence. */
+export function isBlockedUnstartedWake(run: Record<string, unknown>) {
+  return run.status === "cancelled" && run.errorCode === "issue_dependencies_blocked" && run.startedAt === null;
+}

tests/runner-e2e/runner.spec.ts

@@ -791,7 +791,7 @@ for (const execution of executions) {
 
       if (execution.task.flow === "continuation") {
         const continuation = await runContinuationFlow({
-          page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs - 60_000,
+          page, api, fixtures, execution, nonce, secrets, workspacePath, deadlineAt: startedAtMs + deadlineMs - 60_000,
           restart: () => restartIsolatedPaperclipServer({ api, requestId: `continuation-${nonce}`, deadlineAt: startedAtMs + deadlineMs }),
           observe: (currentIssue, currentRuns, checks) => {
             issue = currentIssue; selectedRuns = currentRuns;

tests/runner-e2e/runtime-question-readiness.test.ts

@@ -5,3 +5,14 @@ it("recognizes only a pending native question as an answerable active turn", ()
   expect([...answerableRuntimeRunIds([native])]).toEqual(["live"]);
   expect([...answerableRuntimeRunIds([{ ...native, status: "answered" }, { ...native, payload: {} }])]).toEqual([]);
 });
+
+import { isSingleClaudeQuestion } from "./runtime-question-readiness.js";
+it("allows Claude's optional Other companion but rejects another substantive question", () => {
+  const choice = { id: "choice", answerMode: "single_select" };
+  const other = { id: "field-2-question_0_custom-hash", answerMode: "text", required: false, header: "Other" };
+  expect(isSingleClaudeQuestion([choice])).toBe(true);
+  expect(isSingleClaudeQuestion([choice, other])).toBe(true);
+  expect(isSingleClaudeQuestion([choice, { ...other, required: true }])).toBe(false);
+  expect(isSingleClaudeQuestion([choice, { ...other, id: "organization", header: "Organization" }])).toBe(false);
+  expect(isSingleClaudeQuestion([choice, other, other])).toBe(false);
+});

tests/runner-e2e/runtime-question-readiness.ts

@@ -5,3 +5,10 @@ export function answerableRuntimeRunIds(interactions: ReadonlyArray<Record<strin
     typeof i.sourceRunId === "string" && typeof i.payload?.runtimeRequestId === "string")
     .map((i) => i.sourceRunId));
 }
+
+/** Claude's one question includes the ACP adapter's optional custom-answer
+ * companion field. It is not a second user question. */
+export function isSingleClaudeQuestion(questions: ReadonlyArray<Record<string, any>>) {
+  return questions.length >= 1 && questions.length <= 2 && questions[0]?.answerMode === "single_select" &&
+    questions.slice(1).every(q => q.answerMode === "text" && q.required === false && q.header === "Other" && /_custom-/.test(q.id));
+}