feat(runner): add live eval session entrypoint

packages/paperclip-runner/package.json

@@ -4,6 +4,9 @@
   "private": true,
   "license": "MIT",
   "type": "module",
+  "bin": {
+    "paperclip-runner-eval-session": "./dist/cli/eval-session.js"
+  },
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
@@ -21,6 +24,10 @@
       "types": "./dist/live/index.d.ts",
       "import": "./dist/live/index.js"
     },
+    "./devtools": {
+      "types": "./dist/devtools/index.d.ts",
+      "import": "./dist/devtools/index.js"
+    },
     "./browser": {
       "types": "./dist/browser/index.d.ts",
       "import": "./dist/browser/index.js"

packages/paperclip-runner/src/cli/eval-session.ts

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+import { projectCapabilityDevtools } from "../devtools/index.js";
+import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js";
+import {
+  CapabilityLiveSessionService,
+  reconcileCapabilityLiveUsage,
+  type CreateCapabilityLiveSessionInput,
+} from "../live/live-session.js";
+
+interface EvalSessionRequest {
+  schema: "paperclip-runner/eval-session-request/v1";
+  attemptId: string;
+  prompt: string;
+  model: string;
+  runnerd: { path: string; sha256: string };
+  limits: { turnTimeoutMs: number; maxProviderCalls: number; maxCostNanodollars: number };
+  session: CreateCapabilityLiveSessionInput;
+}
+
+function argument(name: string): string {
+  const index = process.argv.indexOf(name);
+  const value = index < 0 ? undefined : process.argv[index + 1];
+  if (!value) throw new Error(`missing ${name}`);
+  return resolve(value);
+}
+
+async function sha256(path: string): Promise<string> {
+  return createHash("sha256").update(await readFile(path)).digest("hex");
+}
+
+const requestPath = argument("--request");
+const outputPath = argument("--output");
+const request = JSON.parse(await readFile(requestPath, "utf8")) as EvalSessionRequest;
+if (request.schema !== "paperclip-runner/eval-session-request/v1") throw new Error("unsupported request schema");
+const runnerdPath = resolve(request.runnerd.path);
+const actualDigest = await sha256(runnerdPath);
+if (actualDigest !== request.runnerd.sha256.replace(/^sha256:/, "")) {
+  throw new Error(`runnerd digest mismatch: expected ${request.runnerd.sha256}, got sha256:${actualDigest}`);
+}
+
+const service = new CapabilityLiveSessionService({ transportOptions: { runnerBinary: runnerdPath } });
+let sessionId: string | null = null;
+try {
+  const session = await service.create({
+    ...request.session,
+    attemptId: request.attemptId,
+    runId: request.session.runId ?? request.attemptId,
+    requestedModel: request.model,
+    turnTimeoutMs: request.limits.turnTimeoutMs,
+  });
+  sessionId = session.id;
+  const turn = await session.sendMessage(request.prompt);
+  await session.completeAttempt(turn.status === "completed" ? "succeeded" : "failed", `turn_${turn.status}`);
+  const snapshot = session.snapshot();
+  const usage = reconcileCapabilityLiveUsage(snapshot);
+  if (usage.providerCalls > request.limits.maxProviderCalls) throw new Error("provider call limit exceeded");
+  if (usage.costNanodollars > request.limits.maxCostNanodollars) throw new Error("cost limit exceeded");
+  await writeFile(outputPath, `${JSON.stringify({
+    schema: "paperclip-runner/eval-session-artifact/v1",
+    attemptId: request.attemptId,
+    build: PAPERCLIP_RUNNER_BUILD_METADATA,
+    runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },
+    requestedModel: request.model,
+    turn,
+    snapshot,
+    devtools: projectCapabilityDevtools(snapshot),
+    usage,
+  }, null, 2)}\n`);
+} catch (error) {
+  await writeFile(outputPath, `${JSON.stringify({
+    schema: "paperclip-runner/eval-session-artifact/v1",
+    attemptId: request.attemptId,
+    infrastructureError: error instanceof Error ? error.message : String(error),
+    build: PAPERCLIP_RUNNER_BUILD_METADATA,
+    runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },
+    requestedModel: request.model,
+  }, null, 2)}\n`);
+  process.exitCode = 2;
+} finally {
+  if (sessionId !== null) await service.shutdown(sessionId).catch(() => undefined);
+}

packages/paperclip-runner/src/live/live-session.test.ts

@@ -272,6 +272,7 @@ describe("Capability live runnerd and Codex session", () => {
     const session = await service.create({
       runId: "run-live-tool-loop",
       sessionId: "session-live-tool-loop",
+      requestedModel: "gpt-5.4-mini-2026-03-17",
     });
 
     const first = await session.sendMessage("Please report progress on this mock issue.");
@@ -281,6 +282,10 @@ describe("Capability live runnerd and Codex session", () => {
     expect(second.assistantText).toContain("stateRevision");
     expect(first.snapshot.providerThreadId).toBe(second.snapshot.providerThreadId);
     expect(first.snapshot.providerModel).toEqual({ id: "gpt-eval-test", provider: "openai" });
+    expect(first.snapshot.config.requestedModel).toBe("gpt-5.4-mini-2026-03-17");
+    expect(
+      state.transports[0]?.requests.find((entry) => entry.method === "thread/start")?.params.model,
+    ).toBe("gpt-5.4-mini-2026-03-17");
     expect(session.mockState().comments.at(-1)?.body).toBe(
       "Progress persisted through the live Codex tool loop.",
     );

packages/paperclip-runner/src/live/live-session.ts

@@ -88,6 +88,8 @@ export interface CapabilityLiveAuthoritySnapshot {
 
 export interface CapabilityLiveSessionConfigSnapshot {
   workingDirectory: string;
+  /** Exact model requested by the caller; omitted only for legacy interactive sessions. */
+  requestedModel?: string;
   scenario: CapabilitySemanticScenarioPolicy;
   capabilities: string[];
   explicitClaims: string[];
@@ -184,6 +186,7 @@ export interface CapabilityLiveSessionSnapshot {
 export interface CreateCapabilityLiveSessionInput {
   seed?: CapabilityFixtureSeed | CapabilityFixtureState;
   workingDirectory?: string;
+  requestedModel?: string;
   scenario?: CapabilitySemanticScenarioPolicy;
   capabilities?: string[];
   explicitClaims?: string[];
@@ -592,6 +595,9 @@ export class CapabilityLiveSessionService {
       authority,
       config: {
         workingDirectory: input.workingDirectory ?? process.cwd(),
+        ...(input.requestedModel === undefined
+          ? {}
+          : { requestedModel: requireNonEmpty(input.requestedModel, "requested_model") }),
         scenario,
         capabilities,
         explicitClaims: authority.explicitClaims,
@@ -1342,6 +1348,7 @@ export class CapabilityLiveSession {
       }
       const resumed = await this.#transport.request("thread/resume", {
         threadId: this.#providerThreadId,
+        ...(this.#config.requestedModel === undefined ? {} : { model: this.#config.requestedModel }),
         cwd: this.#config.workingDirectory,
         config: createSkilllessCodexThreadConfig(this.#config.workingDirectory),
         permissions: CODEX_PERMISSION_PROFILE,
@@ -1366,6 +1373,7 @@ export class CapabilityLiveSession {
     } else {
       const opened = await this.#transport.request("thread/start", {
         cwd: this.#config.workingDirectory,
+        ...(this.#config.requestedModel === undefined ? {} : { model: this.#config.requestedModel }),
         config: createSkilllessCodexThreadConfig(this.#config.workingDirectory),
         permissions: CODEX_PERMISSION_PROFILE,
         runtimeWorkspaceRoots: [this.#config.workingDirectory],
@@ -1389,6 +1397,7 @@ export class CapabilityLiveSession {
       providerSessionId: this.#providerSessionId,
       mode: "live_codex",
       runner: "paperclip-runnerd",
+      requestedModel: this.#config.requestedModel ?? null,
       controlPlane: "mock",
     });
     this.#pump = this.#pumpNotifications();

packages/paperclip-runner/src/testing.ts

@@ -17,3 +17,4 @@ export * from "./mock-core/capability-mock-control-plane-adapter.js";
 export * from "./protocol/conformance-fixture.js";
 export * from "./live/live-session.js";
 export * from "./live/durable-live-session-store.js";
+export * from "./devtools/index.js";