Qualify legacy Claude skill discovery and await committed question answers

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/runner-full-stack-e2e.yml

@@ -928,7 +928,7 @@ jobs:
 
       - name: Install pinned legacy Claude CLI
         if: matrix.profileId == 'legacy-claude'
-        run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19
+        run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.277
 
       - name: Qualify preinstalled Chrome
         if: needs.authorize.outputs.playwright_channel == 'chrome'

tests/runner-e2e/README.md

@@ -760,3 +760,14 @@ review, external-chat and planning paths retain their existing context. Legacy
 adapter prompts are unchanged.
 
 The ACPX Claude-only `provider-question-bridge` case exercises the provider’s built-in question tool, verifies that its card appears in Paperclip, answers it in the browser, and requires the same paused run to finish with the selected fact. The `accept-while-running` fixture holds the committed card’s creation response until browser acceptance, making the overlap deterministic without changing production behavior.
+
+Local Legacy Claude cells qualify Claude Code `2.1.277` before starting the server.
+If the ambient CLI differs, the harness installs the exact version under the
+attempt's temporary root and prepends that private bin directory to the server's
+PATH. It does not change the developer's global installation. The old workflow
+pin, `2.1.19`, did not discover `.claude/skills` supplied through `--add-dir`;
+a provider-free CLI probe reproduced the missing skill on that version and
+confirmed discovery on `2.1.277`. The workflow pin and local qualifier are checked
+together. This change applies to local cells; Daytona images remain separately pinned.
+Continuation question flows also wait for the submitted interaction's durable
+`answered` state before considering the next checkpoint ready.

tests/runner-e2e/continuation-flow.ts

@@ -3,7 +3,7 @@ import { captureFirstTaskAttachments } from "./first-task-attachments.js";
 import { answerableRuntimeRunIds, isSingleClaudeQuestion } from "./runtime-question-readiness.js";
 import { expect, type Page } from "@playwright/test";
 import path from "node:path";
-import { continuationInitialReady } from "./continuation-readiness.js";
+import { continuationAnswerCommitted, continuationInitialReady } from "./continuation-readiness.js";
 import { captureLoadedContinuation } from "./continuation-screenshot.js";
 import { seedContinuationContext } from "./continuation-workspace.js";
 import { pollUntil, type RunnerApi } from "./api.js";
@@ -63,7 +63,7 @@ export async function runContinuationFlow(input: {
     return { issue, runs };
   }
   let pausedRuntimeRunIds = new Set<string>();
-  async function settle(prior: Set<string>, requireQuestion = false) {
+  async function settle(prior: Set<string>, requireQuestion = false, answeredInteractionId?: string) {
     let stable = "";
     const previousPaused = pausedRuntimeRunIds;
     await pollUntil({
@@ -77,6 +77,7 @@ export async function runContinuationFlow(input: {
       accept: (state) => {
         const paused = answerableRuntimeRunIds(state.interactions);
         const idle =
+          continuationAnswerCommitted(state.interactions, answeredInteractionId) &&
           state.runs.some((r) => !prior.has(r.id) || previousPaused.has(r.id)) &&
           state.runs.every((r) => ["succeeded", "failed", "timed_out", "cancelled"].includes(r.status) ||
             (r.status === "running" && paused.has(r.id))) &&
@@ -184,7 +185,7 @@ export async function runContinuationFlow(input: {
       })
       .last()
       .click();
-    await settle(before);
+    await settle(before, false, questions[0].id);
   }
   async function reply(body: string) {
     const before = new Set(runs.map((r) => r.id));

tests/runner-e2e/continuation-readiness.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { continuationInitialReady } from "./continuation-readiness.js";
+import { continuationAnswerCommitted, continuationInitialReady } from "./continuation-readiness.js";
 import { runnerMatrix } from "./catalog.js";
 
 describe("continuation readiness", () => {
@@ -15,3 +15,11 @@ describe("continuation readiness", () => {
     }
   });
 });
+
+it("waits for the clicked answer to commit instead of grading the original paused state", () => {
+  const card = { id: "submitted", status: "pending" };
+  expect(continuationAnswerCommitted([card], card.id)).toBe(false);
+  expect(continuationAnswerCommitted([{ ...card, id: "different", status: "answered" }], card.id)).toBe(false);
+  expect(continuationAnswerCommitted([{ ...card, status: "cancelled" }], card.id)).toBe(false);
+  expect(continuationAnswerCommitted([{ ...card, status: "answered" }], card.id)).toBe(true);
+});

tests/runner-e2e/continuation-readiness.ts

@@ -2,3 +2,9 @@
 export function continuationInitialReady(interactions: ReadonlyArray<{ kind?: unknown; status?: unknown }>): boolean {
   return interactions.some((i) => i.kind === "ask_user_questions" && i.status === "pending");
 }
+
+/** A successful click can return before the form POST commits. Do not accept
+ * the original paused run as the result of the answer we just submitted. */
+export function continuationAnswerCommitted(interactions: ReadonlyArray<{ id?: unknown; status?: unknown }>, interactionId?: string): boolean {
+  return !interactionId || interactions.some(i => i.id === interactionId && i.status === "answered");
+}

tests/runner-e2e/legacy-claude-cli.test.ts

@@ -0,0 +1,14 @@
+import { readFileSync } from "node:fs";
+import { expect, it } from "vitest";
+import { LEGACY_CLAUDE_CLI_VERSION, qualifiedLegacyClaudeVersion } from "./legacy-claude-cli.js";
+
+it("rejects the old CLI that cannot discover mounted skills and requires the exact qualified version", () => {
+  expect(qualifiedLegacyClaudeVersion("2.1.19 (Claude Code)")).toBe(false);
+  expect(qualifiedLegacyClaudeVersion(`${LEGACY_CLAUDE_CLI_VERSION} (Claude Code)\n`)).toBe(true);
+  expect(qualifiedLegacyClaudeVersion(`warning: ${LEGACY_CLAUDE_CLI_VERSION} (Claude Code)`)).toBe(false);
+});
+it("keeps the workflow installation pin synchronized with local qualification", () => {
+  const workflow = readFileSync(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8");
+  expect(workflow).toContain(`@anthropic-ai/claude-code@${LEGACY_CLAUDE_CLI_VERSION}`);
+  expect(workflow).not.toContain("@anthropic-ai/claude-code@2.1.19");
+});

tests/runner-e2e/legacy-claude-cli.ts

@@ -0,0 +1,27 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import path from "node:path";
+
+// 2.1.19 predates --add-dir skill discovery. Keep local/CI qualification
+// reproducible without replacing a developer's globally installed CLI.
+export const LEGACY_CLAUDE_CLI_VERSION = "2.1.277";
+const execute = promisify(execFile);
+export function qualifiedLegacyClaudeVersion(output: string) {
+  return output.trim() === `${LEGACY_CLAUDE_CLI_VERSION} (Claude Code)`;
+}
+
+export async function qualifyLegacyClaudeCli(temporaryRoot: string, environment: NodeJS.ProcessEnv) {
+  const env = Object.fromEntries(["PATH", "HOME", "TMPDIR", "TEMP", "SystemRoot"]
+    .flatMap(key => environment[key] ? [[key, environment[key]!]] : []));
+  try {
+    const result = await execute("claude", ["--version"], { env, timeout: 15_000 });
+    if (qualifiedLegacyClaudeVersion(result.stdout)) return environment.PATH ?? "";
+  } catch { /* Install the exact fixture version in the attempt's private root. */ }
+  const prefix = path.join(temporaryRoot, "legacy-claude-cli");
+  await execute("npm", ["install", "--prefix", prefix, "--no-save", "--no-package-lock", "--no-audit", "--no-fund",
+    `@anthropic-ai/claude-code@${LEGACY_CLAUDE_CLI_VERSION}`], { env, timeout: 120_000, maxBuffer: 1024 * 1024 });
+  const bin = path.join(prefix, "node_modules", ".bin");
+  const result = await execute(path.join(bin, "claude"), ["--version"], { env, timeout: 15_000 });
+  if (!qualifiedLegacyClaudeVersion(result.stdout)) throw new Error("Legacy Claude CLI version qualification failed");
+  return `${bin}${path.delimiter}${environment.PATH ?? ""}`;
+}

tests/runner-e2e/server.ts

@@ -1,3 +1,4 @@
+import { qualifyLegacyClaudeCli } from "./legacy-claude-cli.js";
 import { spawn, type ChildProcess } from "node:child_process";
 import { createWriteStream } from "node:fs";
 import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
@@ -348,6 +349,10 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
 }
 
 async function supervise() {
+  const executionIds: string[] = JSON.parse(process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]");
+  if (executionIds.some(id => id.includes(".legacy-claude.local."))) {
+    serverEnvironment.PATH = await qualifyLegacyClaudeCli(temporaryRoot, serverEnvironment);
+  }
   const databaseReservation = await prepareRunnerE2EServerConfig({
     temporaryRoot,
     configPath,