diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index d7d64335dc..2c04b81c2f 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -928,7 +928,7 @@ jobs: - name: Install pinned legacy Claude CLI if: matrix.profileId == 'legacy-claude' - run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19 + run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.277 - name: Qualify preinstalled Chrome if: needs.authorize.outputs.playwright_channel == 'chrome' diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index cc82a307c3..5ef2169bd8 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -760,3 +760,14 @@ review, external-chat and planning paths retain their existing context. Legacy adapter prompts are unchanged. The ACPX Claude-only `provider-question-bridge` case exercises the provider’s built-in question tool, verifies that its card appears in Paperclip, answers it in the browser, and requires the same paused run to finish with the selected fact. The `accept-while-running` fixture holds the committed card’s creation response until browser acceptance, making the overlap deterministic without changing production behavior. + +Local Legacy Claude cells qualify Claude Code `2.1.277` before starting the server. +If the ambient CLI differs, the harness installs the exact version under the +attempt's temporary root and prepends that private bin directory to the server's +PATH. It does not change the developer's global installation. The old workflow +pin, `2.1.19`, did not discover `.claude/skills` supplied through `--add-dir`; +a provider-free CLI probe reproduced the missing skill on that version and +confirmed discovery on `2.1.277`. The workflow pin and local qualifier are checked +together. This change applies to local cells; Daytona images remain separately pinned. +Continuation question flows also wait for the submitted interaction's durable +`answered` state before considering the next checkpoint ready. diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index 97f325367a..40681652c3 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -3,7 +3,7 @@ import { captureFirstTaskAttachments } from "./first-task-attachments.js"; import { answerableRuntimeRunIds, isSingleClaudeQuestion } from "./runtime-question-readiness.js"; import { expect, type Page } from "@playwright/test"; import path from "node:path"; -import { continuationInitialReady } from "./continuation-readiness.js"; +import { continuationAnswerCommitted, continuationInitialReady } from "./continuation-readiness.js"; import { captureLoadedContinuation } from "./continuation-screenshot.js"; import { seedContinuationContext } from "./continuation-workspace.js"; import { pollUntil, type RunnerApi } from "./api.js"; @@ -63,7 +63,7 @@ export async function runContinuationFlow(input: { return { issue, runs }; } let pausedRuntimeRunIds = new Set(); - async function settle(prior: Set, requireQuestion = false) { + async function settle(prior: Set, requireQuestion = false, answeredInteractionId?: string) { let stable = ""; const previousPaused = pausedRuntimeRunIds; await pollUntil({ @@ -77,6 +77,7 @@ export async function runContinuationFlow(input: { accept: (state) => { const paused = answerableRuntimeRunIds(state.interactions); const idle = + continuationAnswerCommitted(state.interactions, answeredInteractionId) && state.runs.some((r) => !prior.has(r.id) || previousPaused.has(r.id)) && state.runs.every((r) => ["succeeded", "failed", "timed_out", "cancelled"].includes(r.status) || (r.status === "running" && paused.has(r.id))) && @@ -184,7 +185,7 @@ export async function runContinuationFlow(input: { }) .last() .click(); - await settle(before); + await settle(before, false, questions[0].id); } async function reply(body: string) { const before = new Set(runs.map((r) => r.id)); diff --git a/tests/runner-e2e/continuation-readiness.test.ts b/tests/runner-e2e/continuation-readiness.test.ts index ab5c80d8af..560173b1ea 100644 --- a/tests/runner-e2e/continuation-readiness.test.ts +++ b/tests/runner-e2e/continuation-readiness.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { continuationInitialReady } from "./continuation-readiness.js"; +import { continuationAnswerCommitted, continuationInitialReady } from "./continuation-readiness.js"; import { runnerMatrix } from "./catalog.js"; describe("continuation readiness", () => { @@ -15,3 +15,11 @@ describe("continuation readiness", () => { } }); }); + +it("waits for the clicked answer to commit instead of grading the original paused state", () => { + const card = { id: "submitted", status: "pending" }; + expect(continuationAnswerCommitted([card], card.id)).toBe(false); + expect(continuationAnswerCommitted([{ ...card, id: "different", status: "answered" }], card.id)).toBe(false); + expect(continuationAnswerCommitted([{ ...card, status: "cancelled" }], card.id)).toBe(false); + expect(continuationAnswerCommitted([{ ...card, status: "answered" }], card.id)).toBe(true); +}); diff --git a/tests/runner-e2e/continuation-readiness.ts b/tests/runner-e2e/continuation-readiness.ts index a10108411f..ffc54e8821 100644 --- a/tests/runner-e2e/continuation-readiness.ts +++ b/tests/runner-e2e/continuation-readiness.ts @@ -2,3 +2,9 @@ export function continuationInitialReady(interactions: ReadonlyArray<{ kind?: unknown; status?: unknown }>): boolean { return interactions.some((i) => i.kind === "ask_user_questions" && i.status === "pending"); } + +/** A successful click can return before the form POST commits. Do not accept + * the original paused run as the result of the answer we just submitted. */ +export function continuationAnswerCommitted(interactions: ReadonlyArray<{ id?: unknown; status?: unknown }>, interactionId?: string): boolean { + return !interactionId || interactions.some(i => i.id === interactionId && i.status === "answered"); +} diff --git a/tests/runner-e2e/legacy-claude-cli.test.ts b/tests/runner-e2e/legacy-claude-cli.test.ts new file mode 100644 index 0000000000..a6a0032ee5 --- /dev/null +++ b/tests/runner-e2e/legacy-claude-cli.test.ts @@ -0,0 +1,14 @@ +import { readFileSync } from "node:fs"; +import { expect, it } from "vitest"; +import { LEGACY_CLAUDE_CLI_VERSION, qualifiedLegacyClaudeVersion } from "./legacy-claude-cli.js"; + +it("rejects the old CLI that cannot discover mounted skills and requires the exact qualified version", () => { + expect(qualifiedLegacyClaudeVersion("2.1.19 (Claude Code)")).toBe(false); + expect(qualifiedLegacyClaudeVersion(`${LEGACY_CLAUDE_CLI_VERSION} (Claude Code)\n`)).toBe(true); + expect(qualifiedLegacyClaudeVersion(`warning: ${LEGACY_CLAUDE_CLI_VERSION} (Claude Code)`)).toBe(false); +}); +it("keeps the workflow installation pin synchronized with local qualification", () => { + const workflow = readFileSync(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8"); + expect(workflow).toContain(`@anthropic-ai/claude-code@${LEGACY_CLAUDE_CLI_VERSION}`); + expect(workflow).not.toContain("@anthropic-ai/claude-code@2.1.19"); +}); diff --git a/tests/runner-e2e/legacy-claude-cli.ts b/tests/runner-e2e/legacy-claude-cli.ts new file mode 100644 index 0000000000..a6b34eb05d --- /dev/null +++ b/tests/runner-e2e/legacy-claude-cli.ts @@ -0,0 +1,27 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import path from "node:path"; + +// 2.1.19 predates --add-dir skill discovery. Keep local/CI qualification +// reproducible without replacing a developer's globally installed CLI. +export const LEGACY_CLAUDE_CLI_VERSION = "2.1.277"; +const execute = promisify(execFile); +export function qualifiedLegacyClaudeVersion(output: string) { + return output.trim() === `${LEGACY_CLAUDE_CLI_VERSION} (Claude Code)`; +} + +export async function qualifyLegacyClaudeCli(temporaryRoot: string, environment: NodeJS.ProcessEnv) { + const env = Object.fromEntries(["PATH", "HOME", "TMPDIR", "TEMP", "SystemRoot"] + .flatMap(key => environment[key] ? [[key, environment[key]!]] : [])); + try { + const result = await execute("claude", ["--version"], { env, timeout: 15_000 }); + if (qualifiedLegacyClaudeVersion(result.stdout)) return environment.PATH ?? ""; + } catch { /* Install the exact fixture version in the attempt's private root. */ } + const prefix = path.join(temporaryRoot, "legacy-claude-cli"); + await execute("npm", ["install", "--prefix", prefix, "--no-save", "--no-package-lock", "--no-audit", "--no-fund", + `@anthropic-ai/claude-code@${LEGACY_CLAUDE_CLI_VERSION}`], { env, timeout: 120_000, maxBuffer: 1024 * 1024 }); + const bin = path.join(prefix, "node_modules", ".bin"); + const result = await execute(path.join(bin, "claude"), ["--version"], { env, timeout: 15_000 }); + if (!qualifiedLegacyClaudeVersion(result.stdout)) throw new Error("Legacy Claude CLI version qualification failed"); + return `${bin}${path.delimiter}${environment.PATH ?? ""}`; +} diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index 6c5ff445b2..03870a4daa 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -1,3 +1,4 @@ +import { qualifyLegacyClaudeCli } from "./legacy-claude-cli.js"; import { spawn, type ChildProcess } from "node:child_process"; import { createWriteStream } from "node:fs"; import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; @@ -348,6 +349,10 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) { } async function supervise() { + const executionIds: string[] = JSON.parse(process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]"); + if (executionIds.some(id => id.includes(".legacy-claude.local."))) { + serverEnvironment.PATH = await qualifyLegacyClaudeCli(temporaryRoot, serverEnvironment); + } const databaseReservation = await prepareRunnerE2EServerConfig({ temporaryRoot, configPath,