diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 99d7c87ffa5..bc7c587f6ce 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -4,6 +4,9 @@ "private": true, "license": "MIT", "type": "module", + "bin": { + "paperclip-runner-eval-session": "./dist/cli/eval-session.js" + }, "exports": { ".": { "types": "./dist/index.d.ts", @@ -21,6 +24,10 @@ "types": "./dist/live/index.d.ts", "import": "./dist/live/index.js" }, + "./devtools": { + "types": "./dist/devtools/index.d.ts", + "import": "./dist/devtools/index.js" + }, "./browser": { "types": "./dist/browser/index.d.ts", "import": "./dist/browser/index.js" diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts new file mode 100644 index 00000000000..bc06fbab3d9 --- /dev/null +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { projectCapabilityDevtools } from "../devtools/index.js"; +import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js"; +import { + CapabilityLiveSessionService, + reconcileCapabilityLiveUsage, + type CreateCapabilityLiveSessionInput, +} from "../live/live-session.js"; + +interface EvalSessionRequest { + schema: "paperclip-runner/eval-session-request/v1"; + attemptId: string; + prompt: string; + model: string; + runnerd: { path: string; sha256: string }; + limits: { turnTimeoutMs: number; maxProviderCalls: number; maxCostNanodollars: number }; + session: CreateCapabilityLiveSessionInput; +} + +function argument(name: string): string { + const index = process.argv.indexOf(name); + const value = index < 0 ? undefined : process.argv[index + 1]; + if (!value) throw new Error(`missing ${name}`); + return resolve(value); +} + +async function sha256(path: string): Promise { + return createHash("sha256").update(await readFile(path)).digest("hex"); +} + +const requestPath = argument("--request"); +const outputPath = argument("--output"); +const request = JSON.parse(await readFile(requestPath, "utf8")) as EvalSessionRequest; +if (request.schema !== "paperclip-runner/eval-session-request/v1") throw new Error("unsupported request schema"); +const runnerdPath = resolve(request.runnerd.path); +const actualDigest = await sha256(runnerdPath); +if (actualDigest !== request.runnerd.sha256.replace(/^sha256:/, "")) { + throw new Error(`runnerd digest mismatch: expected ${request.runnerd.sha256}, got sha256:${actualDigest}`); +} + +const service = new CapabilityLiveSessionService({ transportOptions: { runnerBinary: runnerdPath } }); +let sessionId: string | null = null; +try { + const session = await service.create({ + ...request.session, + attemptId: request.attemptId, + runId: request.session.runId ?? request.attemptId, + requestedModel: request.model, + turnTimeoutMs: request.limits.turnTimeoutMs, + }); + sessionId = session.id; + const turn = await session.sendMessage(request.prompt); + await session.completeAttempt(turn.status === "completed" ? "succeeded" : "failed", `turn_${turn.status}`); + const snapshot = session.snapshot(); + const usage = reconcileCapabilityLiveUsage(snapshot); + if (usage.providerCalls > request.limits.maxProviderCalls) throw new Error("provider call limit exceeded"); + if (usage.costNanodollars > request.limits.maxCostNanodollars) throw new Error("cost limit exceeded"); + await writeFile(outputPath, `${JSON.stringify({ + schema: "paperclip-runner/eval-session-artifact/v1", + attemptId: request.attemptId, + build: PAPERCLIP_RUNNER_BUILD_METADATA, + runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, + requestedModel: request.model, + turn, + snapshot, + devtools: projectCapabilityDevtools(snapshot), + usage, + }, null, 2)}\n`); +} catch (error) { + await writeFile(outputPath, `${JSON.stringify({ + schema: "paperclip-runner/eval-session-artifact/v1", + attemptId: request.attemptId, + infrastructureError: error instanceof Error ? error.message : String(error), + build: PAPERCLIP_RUNNER_BUILD_METADATA, + runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, + requestedModel: request.model, + }, null, 2)}\n`); + process.exitCode = 2; +} finally { + if (sessionId !== null) await service.shutdown(sessionId).catch(() => undefined); +} diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 426e6d8a39c..7e3e4413e26 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -272,6 +272,7 @@ describe("Capability live runnerd and Codex session", () => { const session = await service.create({ runId: "run-live-tool-loop", sessionId: "session-live-tool-loop", + requestedModel: "gpt-5.4-mini-2026-03-17", }); const first = await session.sendMessage("Please report progress on this mock issue."); @@ -281,6 +282,10 @@ describe("Capability live runnerd and Codex session", () => { expect(second.assistantText).toContain("stateRevision"); expect(first.snapshot.providerThreadId).toBe(second.snapshot.providerThreadId); expect(first.snapshot.providerModel).toEqual({ id: "gpt-eval-test", provider: "openai" }); + expect(first.snapshot.config.requestedModel).toBe("gpt-5.4-mini-2026-03-17"); + expect( + state.transports[0]?.requests.find((entry) => entry.method === "thread/start")?.params.model, + ).toBe("gpt-5.4-mini-2026-03-17"); expect(session.mockState().comments.at(-1)?.body).toBe( "Progress persisted through the live Codex tool loop.", ); diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index b6bed24c8f2..48de2bc6eff 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -88,6 +88,8 @@ export interface CapabilityLiveAuthoritySnapshot { export interface CapabilityLiveSessionConfigSnapshot { workingDirectory: string; + /** Exact model requested by the caller; omitted only for legacy interactive sessions. */ + requestedModel?: string; scenario: CapabilitySemanticScenarioPolicy; capabilities: string[]; explicitClaims: string[]; @@ -184,6 +186,7 @@ export interface CapabilityLiveSessionSnapshot { export interface CreateCapabilityLiveSessionInput { seed?: CapabilityFixtureSeed | CapabilityFixtureState; workingDirectory?: string; + requestedModel?: string; scenario?: CapabilitySemanticScenarioPolicy; capabilities?: string[]; explicitClaims?: string[]; @@ -592,6 +595,9 @@ export class CapabilityLiveSessionService { authority, config: { workingDirectory: input.workingDirectory ?? process.cwd(), + ...(input.requestedModel === undefined + ? {} + : { requestedModel: requireNonEmpty(input.requestedModel, "requested_model") }), scenario, capabilities, explicitClaims: authority.explicitClaims, @@ -1342,6 +1348,7 @@ export class CapabilityLiveSession { } const resumed = await this.#transport.request("thread/resume", { threadId: this.#providerThreadId, + ...(this.#config.requestedModel === undefined ? {} : { model: this.#config.requestedModel }), cwd: this.#config.workingDirectory, config: createSkilllessCodexThreadConfig(this.#config.workingDirectory), permissions: CODEX_PERMISSION_PROFILE, @@ -1366,6 +1373,7 @@ export class CapabilityLiveSession { } else { const opened = await this.#transport.request("thread/start", { cwd: this.#config.workingDirectory, + ...(this.#config.requestedModel === undefined ? {} : { model: this.#config.requestedModel }), config: createSkilllessCodexThreadConfig(this.#config.workingDirectory), permissions: CODEX_PERMISSION_PROFILE, runtimeWorkspaceRoots: [this.#config.workingDirectory], @@ -1389,6 +1397,7 @@ export class CapabilityLiveSession { providerSessionId: this.#providerSessionId, mode: "live_codex", runner: "paperclip-runnerd", + requestedModel: this.#config.requestedModel ?? null, controlPlane: "mock", }); this.#pump = this.#pumpNotifications(); diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts index 7ac52f9a48d..6b782074598 100644 --- a/packages/paperclip-runner/src/testing.ts +++ b/packages/paperclip-runner/src/testing.ts @@ -17,3 +17,4 @@ export * from "./mock-core/capability-mock-control-plane-adapter.js"; export * from "./protocol/conformance-fixture.js"; export * from "./live/live-session.js"; export * from "./live/durable-live-session-store.js"; +export * from "./devtools/index.js";