diff --git a/.github/workflows/runner-live-evals.yml b/.github/workflows/runner-live-evals.yml index c88bc03c56a..93f92d1b483 100644 --- a/.github/workflows/runner-live-evals.yml +++ b/.github/workflows/runner-live-evals.yml @@ -51,6 +51,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} PAPERCLIP_EVAL_BASELINE_READY: "true" + PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }} PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12" PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-day-v1 run: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals diff --git a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs index eca43745078..ae96a5daea4 100644 --- a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs +++ b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs @@ -17,8 +17,10 @@ const historyDirectory = resolve(process.env.PAPERCLIP_EVAL_HISTORY_DIR ?? resol await mkdir(outputDirectory, { recursive: true }); function safeBundleId(schedule) { + const runnerBuild = process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version; const identity = JSON.stringify({ runnerVersion: packageManifest.version, + runnerBuild, promptPolicyId: "runner-live-workflow-v1", seed: schedule.seed, candidates: schedule.candidates.map(({ id, adapter, model, reasoningEffort }) => ({ id, adapter, model, reasoningEffort })), @@ -125,6 +127,7 @@ if (mode === "nightly") { bundle: { id: bundleId, runnerVersion: packageManifest.version, + runnerBuild: process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version, promptPolicyId: "runner-live-workflow-v1", providerVersions, scheduleSeed: schedule.seed, diff --git a/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts b/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts index 121208e97aa..76fb0e238ac 100644 --- a/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts +++ b/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts @@ -3,7 +3,7 @@ import { createInterface } from "node:readline"; import { readFileSync } from "node:fs"; import { resolve } from "node:path"; -import type { HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; +import type { HarnessRuntimeRequestResolution, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; import { createCodexTaskEnvelope } from "../contracts/codex.js"; import { OpenCodeServerDriver } from "../drivers/opencode/opencode-server-driver.js"; import { parseNativeRuntimeContext } from "../contracts/runtime-context.js"; @@ -177,6 +177,21 @@ async function pumpEvents(opened: HarnessSession): Promise { result: payload, }, }); + } else if (event.eventType === "runtime_request.created") { + const request = record(payload.request); + const requestId = text(request.requestId); + const turnId = text(request.turnId, text(event.turnId, activeTurnId ?? "")); + if (!requestId || !turnId || !opened.resolveRuntimeRequest) { + throw new Error("OpenCode runtime request is not resolvable"); + } + const controllerResponse = record(await requestController("paperclip/runtimeRequest", { + request, + })); + await opened.resolveRuntimeRequest({ + requestId, + turnId, + resolution: record(controllerResponse.resolution) as HarnessRuntimeRequestResolution, + }); } else if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) { const status = event.eventType.slice("turn.".length); send({ method: "turn/completed", params: { threadId: opened.ids().driverSessionId, turnId: event.turnId, turn: { id: event.turnId, status } } }); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index c803f34f354..10cf454cfcb 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -334,6 +334,69 @@ describe("OpenCodeServerDriver", () => { await recovered!.session!.close({ reason: "recovery-test" }); }); + it("normalizes native question.asked events and replies with ordered OpenCode answer arrays", async () => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-question-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-question-workspace-")); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + runtimeDirectory: root, + command: fixture, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" }, + }); + const session = await driver.openSession({ + runId: "run-native-question", + normalizedSessionId: "native-question", + workingDirectory: workspace, + }); + const { turnId } = await session.startTurn({ message: { role: "user", text: "native-question" } }); + const iterator = session.events()[Symbol.asyncIterator](); + let requestEvent: Awaited>["value"] | null = null; + for (let count = 0; count < 30; count += 1) { + const event = await iterator.next(); + if (event.done) break; + if (event.value.eventType === "runtime_request.created") { + requestEvent = event.value; + break; + } + } + expect(requestEvent?.payload).toMatchObject({ + request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + type: "input", + input: { + schema: "paperclip.question_set.v1", + questions: [ + { id: "environment", answerMode: "single_select", customAnswer: { enabled: true } }, + { id: "regions", answerMode: "multi_select" }, + ], + }, + }, + }); + await session.resolveRuntimeRequest?.({ + requestId: "question-native-1", + turnId, + resolution: { + action: "submit", + response: { + schema: "paperclip.question_response.v1", + answers: { + environment: { selectedOptionIds: ["option-1"] }, + regions: { selectedOptionIds: ["option-1", "option-2"] }, + }, + }, + }, + }); + const reply = JSON.parse(await readFile(join(root, "native-question", "data", "fake-question-reply.json"), "utf8")); + expect(reply).toEqual({ + url: expect.stringContaining(`directory=${encodeURIComponent(workspace)}`), + body: { answers: [["Staging"], ["US", "EU"]] }, + }); + await session.close({ reason: "test" }); + }); + it("clears a stale active turn that already has a persisted terminal fingerprint", async () => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-stale-turn-")); diff --git a/packages/paperclip-runner/src/eval/workflow-report.ts b/packages/paperclip-runner/src/eval/workflow-report.ts index 559744bb897..6c9e1989bc1 100644 --- a/packages/paperclip-runner/src/eval/workflow-report.ts +++ b/packages/paperclip-runner/src/eval/workflow-report.ts @@ -10,6 +10,7 @@ export const RUNNER_WORKFLOW_REPORT_SCHEMA = "paperclip.runner.workflow-eval-rep export interface RunnerWorkflowReportBundle { id: string; runnerVersion: string; + runnerBuild?: string; promptPolicyId: string; providerVersions: Record; scheduleSeed?: string;