test(runner): extend live eval and OpenCode coverage

.github/workflows/runner-live-evals.yml

@@ -51,6 +51,7 @@ jobs:
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           PAPERCLIP_EVAL_BASELINE_READY: "true"
+          PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }}
           PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12"
           PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-day-v1
         run: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals

packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs

@@ -17,8 +17,10 @@ const historyDirectory = resolve(process.env.PAPERCLIP_EVAL_HISTORY_DIR ?? resol
 await mkdir(outputDirectory, { recursive: true });
 
 function safeBundleId(schedule) {
+  const runnerBuild = process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version;
   const identity = JSON.stringify({
     runnerVersion: packageManifest.version,
+    runnerBuild,
     promptPolicyId: "runner-live-workflow-v1",
     seed: schedule.seed,
     candidates: schedule.candidates.map(({ id, adapter, model, reasoningEffort }) => ({ id, adapter, model, reasoningEffort })),
@@ -125,6 +127,7 @@ if (mode === "nightly") {
     bundle: {
       id: bundleId,
       runnerVersion: packageManifest.version,
+      runnerBuild: process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version,
       promptPolicyId: "runner-live-workflow-v1",
       providerVersions,
       scheduleSeed: schedule.seed,

packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts

@@ -3,7 +3,7 @@ import { createInterface } from "node:readline";
 import { readFileSync } from "node:fs";
 import { resolve } from "node:path";
 
-import type { HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js";
+import type { HarnessRuntimeRequestResolution, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js";
 import { createCodexTaskEnvelope } from "../contracts/codex.js";
 import { OpenCodeServerDriver } from "../drivers/opencode/opencode-server-driver.js";
 import { parseNativeRuntimeContext } from "../contracts/runtime-context.js";
@@ -177,6 +177,21 @@ async function pumpEvents(opened: HarnessSession): Promise<void> {
           result: payload,
         },
       });
+    } else if (event.eventType === "runtime_request.created") {
+      const request = record(payload.request);
+      const requestId = text(request.requestId);
+      const turnId = text(request.turnId, text(event.turnId, activeTurnId ?? ""));
+      if (!requestId || !turnId || !opened.resolveRuntimeRequest) {
+        throw new Error("OpenCode runtime request is not resolvable");
+      }
+      const controllerResponse = record(await requestController("paperclip/runtimeRequest", {
+        request,
+      }));
+      await opened.resolveRuntimeRequest({
+        requestId,
+        turnId,
+        resolution: record(controllerResponse.resolution) as HarnessRuntimeRequestResolution,
+      });
     } else if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) {
       const status = event.eventType.slice("turn.".length);
       send({ method: "turn/completed", params: { threadId: opened.ids().driverSessionId, turnId: event.turnId, turn: { id: event.turnId, status } } });

packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts

@@ -334,6 +334,69 @@ describe("OpenCodeServerDriver", () => {
     await recovered!.session!.close({ reason: "recovery-test" });
   });
 
+  it("normalizes native question.asked events and replies with ordered OpenCode answer arrays", async () => {
+    await chmod(fixture, 0o755);
+    const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-question-"));
+    const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-question-workspace-"));
+    roots.push(root, workspace);
+    const driver = new OpenCodeServerDriver({
+      model: "openrouter/deepseek/deepseek-v4-flash-0731",
+      runtimeDirectory: root,
+      command: fixture,
+      environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
+    });
+    const session = await driver.openSession({
+      runId: "run-native-question",
+      normalizedSessionId: "native-question",
+      workingDirectory: workspace,
+    });
+    const { turnId } = await session.startTurn({ message: { role: "user", text: "native-question" } });
+    const iterator = session.events()[Symbol.asyncIterator]();
+    let requestEvent: Awaited<ReturnType<typeof iterator.next>>["value"] | null = null;
+    for (let count = 0; count < 30; count += 1) {
+      const event = await iterator.next();
+      if (event.done) break;
+      if (event.value.eventType === "runtime_request.created") {
+        requestEvent = event.value;
+        break;
+      }
+    }
+    expect(requestEvent?.payload).toMatchObject({
+      request: {
+        schema: "paperclip.runtime_request.v2",
+        requestKind: "runtime",
+        type: "input",
+        input: {
+          schema: "paperclip.question_set.v1",
+          questions: [
+            { id: "environment", answerMode: "single_select", customAnswer: { enabled: true } },
+            { id: "regions", answerMode: "multi_select" },
+          ],
+        },
+      },
+    });
+    await session.resolveRuntimeRequest?.({
+      requestId: "question-native-1",
+      turnId,
+      resolution: {
+        action: "submit",
+        response: {
+          schema: "paperclip.question_response.v1",
+          answers: {
+            environment: { selectedOptionIds: ["option-1"] },
+            regions: { selectedOptionIds: ["option-1", "option-2"] },
+          },
+        },
+      },
+    });
+    const reply = JSON.parse(await readFile(join(root, "native-question", "data", "fake-question-reply.json"), "utf8"));
+    expect(reply).toEqual({
+      url: expect.stringContaining(`directory=${encodeURIComponent(workspace)}`),
+      body: { answers: [["Staging"], ["US", "EU"]] },
+    });
+    await session.close({ reason: "test" });
+  });
+
   it("clears a stale active turn that already has a persisted terminal fingerprint", async () => {
     await chmod(fixture, 0o755);
     const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-stale-turn-"));

packages/paperclip-runner/src/eval/workflow-report.ts

@@ -10,6 +10,7 @@ export const RUNNER_WORKFLOW_REPORT_SCHEMA = "paperclip.runner.workflow-eval-rep
 export interface RunnerWorkflowReportBundle {
   id: string;
   runnerVersion: string;
+  runnerBuild?: string;
   promptPolicyId: string;
   providerVersions: Record<string, string>;
   scheduleSeed?: string;