diff --git a/cli/src/__tests__/bench.test.ts b/cli/src/__tests__/bench.test.ts new file mode 100644 index 00000000000..7a671b7a35b --- /dev/null +++ b/cli/src/__tests__/bench.test.ts @@ -0,0 +1,345 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + loadBenchTasks, + normalizeBenchRunner, + normalizeBenchTask, + renderBenchIssueMarkdown, + runPaperclipBench, +} from "../commands/bench.js"; + +describe("normalizeBenchTask", () => { + it("accepts MissionBench-style snake_case fields", () => { + const task = normalizeBenchTask( + { + id: "missionbench-001", + title: "Add audit logs", + repo: "github.com/paperclipai/missionbench-saas", + starting_commit: "abc123", + problem_statement: "Add organization-level audit logs.", + time_budget_minutes: 180, + token_budget_usd: 20, + success_checks: ["npm test", "npm run typecheck"], + human_rubric: { architecture: 5, test_quality: 5 }, + required_artifacts: ["implementation patch", "risk notes"], + }, + { lineLabel: "tasks.jsonl:1" }, + ); + + expect(task).toMatchObject({ + id: "missionbench-001", + title: "Add audit logs", + problemStatement: "Add organization-level audit logs.", + repo: { + url: "github.com/paperclipai/missionbench-saas", + baseCommit: "abc123", + }, + maxWallClockMinutes: 180, + maxCostUsd: 20, + successChecks: ["npm test", "npm run typecheck"], + requiredArtifacts: ["implementation patch", "risk notes"], + }); + expect(task.humanRubric).toEqual([ + { name: "architecture", maxScore: 5 }, + { name: "test_quality", maxScore: 5 }, + ]); + }); + + it("rejects tasks without a problem statement", () => { + expect(() => + normalizeBenchTask( + { + id: "missing-problem", + title: "Missing problem", + }, + { lineLabel: "tasks.jsonl:2" }, + ), + ).toThrow(/requires problemStatement/i); + }); + + it("keeps an explicit issue reference for reuse", () => { + const task = normalizeBenchTask( + { + id: "missionbench-002", + title: "Reuse benchmark issue", + problem_statement: "Continue work in an existing benchmark issue.", + paperclip_issue_id: "PAP-42", + }, + { lineLabel: "tasks.jsonl:3" }, + ); + + expect(task.paperclipIssueRef).toBe("PAP-42"); + }); +}); + +describe("normalizeBenchRunner", () => { + it("accepts prepare and paperclip", () => { + expect(normalizeBenchRunner(undefined)).toBe("prepare"); + expect(normalizeBenchRunner("prepare")).toBe("prepare"); + expect(normalizeBenchRunner("paperclip")).toBe("paperclip"); + }); + + it("rejects unsupported runners", () => { + expect(() => normalizeBenchRunner("live")).toThrow(/Unsupported bench runner/i); + }); +}); + +describe("renderBenchIssueMarkdown", () => { + it("renders a Paperclip-ready task prompt", () => { + const markdown = renderBenchIssueMarkdown( + { + id: "missionbench-001", + title: "Add audit logs", + problemStatement: "Add organization-level audit logs.", + repo: { url: "https://example.com/repo.git", baseCommit: "abc123" }, + maxWallClockMinutes: 120, + maxCostUsd: 10, + successChecks: ["npm test"], + humanRubric: [{ name: "architecture", maxScore: 5 }], + requiredArtifacts: ["final PR summary"], + }, + { + benchmark: "missionbench", + runId: "run-1", + model: "anthropic/claude-sonnet-4.5", + paperclipVersion: "current", + }, + ); + + expect(markdown).toContain("# Add audit logs"); + expect(markdown).toContain("- Task ID: `missionbench-001`"); + expect(markdown).toContain("- URL: https://example.com/repo.git"); + expect(markdown).toContain("- npm test"); + expect(markdown).toContain("- architecture (5 pts)"); + }); +}); + +describe("runPaperclipBench", () => { + it("writes manifest, per-task prompts, traces, and results", async () => { + const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-test-")); + try { + const tasksPath = path.join(tmpDir, "tasks.jsonl"); + const outputDir = path.join(tmpDir, "out"); + await writeFile( + tasksPath, + `${JSON.stringify({ + id: "task-1", + title: "Fix checkout", + problemStatement: "Make checkout idempotent.", + successChecks: ["pnpm test"], + })}\n`, + "utf8", + ); + + const manifest = await runPaperclipBench({ + tasksPath, + benchmark: "paperclip_native", + outputDir, + runId: "test-run", + paperclipVersion: "current", + model: "noop", + runner: "prepare", + clean: true, + }); + + expect(manifest.taskCount).toBe(1); + expect(manifest.tasks[0]?.issuePath).toBe("tasks/task-1/issue.md"); + + const issueMarkdown = await readFile(path.join(outputDir, "tasks/task-1/issue.md"), "utf8"); + expect(issueMarkdown).toContain("Make checkout idempotent."); + + const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8"); + expect(results).toContain('"status":"prepared"'); + + const reloadedTasks = await loadBenchTasks(tasksPath); + expect(reloadedTasks.map((task) => task.id)).toEqual(["task-1"]); + } finally { + await rm(tmpDir, { recursive: true, force: true }); + } + }); + + it("records live Paperclip runner outputs", async () => { + const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-live-test-")); + try { + const tasksPath = path.join(tmpDir, "tasks.jsonl"); + const outputDir = path.join(tmpDir, "out"); + await writeFile( + tasksPath, + `${JSON.stringify({ + id: "task-live-1", + title: "Ship audit logs", + problemStatement: "Add audit log support.", + successChecks: ["pnpm test"], + })}\n`, + "utf8", + ); + + const manifest = await runPaperclipBench({ + tasksPath, + benchmark: "paperclip_native", + outputDir, + runId: "live-run", + paperclipVersion: "current", + model: "noop", + runner: "paperclip", + clean: true, + issueProvisioner: async () => ({ + source: "created", + issue: { + id: "issue-1", + companyId: "company-1", + projectId: null, + projectWorkspaceId: null, + goalId: null, + parentId: null, + title: "Ship audit logs", + description: null, + status: "todo", + priority: "medium", + assigneeAgentId: "agent-1", + assigneeUserId: null, + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + createdByAgentId: null, + createdByUserId: null, + issueNumber: 1, + identifier: "PAP-1", + requestDepth: 0, + billingCode: null, + assigneeAdapterOverrides: null, + executionWorkspaceId: null, + executionWorkspacePreference: null, + executionWorkspaceSettings: null, + startedAt: null, + completedAt: null, + cancelledAt: null, + hiddenAt: null, + createdAt: new Date("2026-04-24T12:00:00Z"), + updatedAt: new Date("2026-04-24T12:00:00Z"), + }, + }), + runLiveTask: async ({ taskDir, issue }) => { + await writeFile(path.join(taskDir, "patch.diff"), "diff --git a/file b/file\n", "utf8"); + return { + issue: { + ...issue, + status: "done", + executionRunId: "run-2", + updatedAt: new Date("2026-04-24T12:02:00Z"), + }, + runIds: ["run-1", "run-2"], + status: "done", + patchPath: "tasks/task-live-1/patch.diff", + }; + }, + }); + + expect(manifest.runner).toBe("paperclip"); + expect(manifest.tasks[0]).toMatchObject({ + status: "done", + patchPath: "tasks/task-live-1/patch.diff", + paperclipRunIds: ["run-1", "run-2"], + paperclipIssue: { + id: "issue-1", + identifier: "PAP-1", + status: "done", + source: "created", + }, + }); + + const trace = await readFile(path.join(outputDir, "tasks/task-live-1/trace.json"), "utf8"); + expect(trace).toContain('"paperclipRunIds": ['); + expect(trace).toContain('"patchPath": "tasks/task-live-1/patch.diff"'); + + const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8"); + expect(results).toContain('"status":"done"'); + expect(results).toContain('"paperclipRunIds":["run-1","run-2"]'); + expect(results).toContain('"patchPath":"tasks/task-live-1/patch.diff"'); + } finally { + await rm(tmpDir, { recursive: true, force: true }); + } + }); + + it("writes failed live-run results when the Paperclip runner errors", async () => { + const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-live-fail-test-")); + try { + const tasksPath = path.join(tmpDir, "tasks.jsonl"); + const outputDir = path.join(tmpDir, "out"); + await writeFile( + tasksPath, + `${JSON.stringify({ + id: "task-live-2", + title: "Fail audit logs", + problemStatement: "Add audit log support.", + successChecks: ["pnpm test"], + })}\n`, + "utf8", + ); + + const manifest = await runPaperclipBench({ + tasksPath, + benchmark: "paperclip_native", + outputDir, + runId: "live-fail-run", + paperclipVersion: "current", + model: "noop", + runner: "paperclip", + clean: true, + continueOnError: true, + issueProvisioner: async () => ({ + source: "reused", + issue: { + id: "issue-2", + companyId: "company-1", + projectId: null, + projectWorkspaceId: null, + goalId: null, + parentId: null, + title: "Fail audit logs", + description: null, + status: "in_progress", + priority: "medium", + assigneeAgentId: "agent-1", + assigneeUserId: null, + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + createdByAgentId: null, + createdByUserId: null, + issueNumber: 2, + identifier: "PAP-2", + requestDepth: 0, + billingCode: null, + assigneeAdapterOverrides: null, + executionWorkspaceId: null, + executionWorkspacePreference: null, + executionWorkspaceSettings: null, + startedAt: null, + completedAt: null, + cancelledAt: null, + hiddenAt: null, + createdAt: new Date("2026-04-24T12:00:00Z"), + updatedAt: new Date("2026-04-24T12:00:00Z"), + }, + }), + runLiveTask: async () => { + throw new Error("heartbeat invoke failed"); + }, + }); + + expect(manifest.tasks[0]?.status).toBe("failed"); + + const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8"); + expect(results).toContain('"status":"failed"'); + expect(results).toContain('"failureStage":"paperclip_runner"'); + expect(results).toContain('"notes":"heartbeat invoke failed"'); + } finally { + await rm(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/cli/src/commands/bench.ts b/cli/src/commands/bench.ts new file mode 100644 index 00000000000..2840798fc7c --- /dev/null +++ b/cli/src/commands/bench.ts @@ -0,0 +1,1277 @@ +import { spawnSync } from "node:child_process"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import pc from "picocolors"; +import { Command } from "commander"; +import type { AgentWakeupResponse, HeartbeatRun, Issue } from "@paperclipai/shared"; +import { + addCommonClientOptions, + handleCommandError, + printOutput, + resolveCommandContext, + type BaseClientOptions, +} from "./client/common.js"; +import { ApiRequestError } from "../client/http.js"; + +export interface BenchRepoSpec { + url?: string; + path?: string; + baseCommit?: string; +} + +export interface BenchRubricItem { + name: string; + description?: string; + maxScore?: number; +} + +export interface BenchTask { + id: string; + title: string; + problemStatement: string; + repo?: BenchRepoSpec; + paperclipIssueRef?: string; + maxWallClockMinutes?: number; + maxCostUsd?: number; + successChecks: string[]; + humanRubric: BenchRubricItem[]; + requiredArtifacts: string[]; + metadata?: Record; +} + +export interface BenchManifestTask { + id: string; + title: string; + issuePath: string; + taskPath: string; + tracePath: string; + patchPath?: string; + workspacePath?: string; + repo?: BenchRepoSpec; + successChecks: string[]; + requiredArtifacts: string[]; + paperclipRunIds?: string[]; + status?: BenchTaskStatus; + paperclipIssue?: { + id: string; + identifier?: string; + status?: string; + source?: "created" | "reused"; + }; +} + +export type BenchRunner = "prepare" | "paperclip"; +export type BenchTaskStatus = "prepared" | "issue_created" | "issue_reused" | Issue["status"] | "failed"; + +export interface BenchManifest { + schemaVersion: 1; + runId: string; + benchmark: string; + paperclipVersion: string; + model: string; + runner: BenchRunner; + createdAt: string; + outputDir: string; + maxCostUsd?: number; + maxWallClockMinutes?: number; + taskCount: number; + tasks: BenchManifestTask[]; +} + +export interface BenchResult { + runId: string; + benchmark: string; + taskId: string; + status: BenchTaskStatus; + issuePath: string; + taskPath: string; + tracePath: string; + patchPath?: string; + workspacePath?: string; + paperclipIssueId?: string; + paperclipIssueIdentifier?: string; + paperclipRunIds?: string[]; + failureStage?: string | null; + notes?: string; +} + +interface BenchRunOptions extends BaseClientOptions { + tasks: string; + benchmark?: string; + output?: string; + runId?: string; + paperclipVersion?: string; + model?: string; + runner?: string; + maxCostUsd?: string; + maxWallClockMinutes?: string; + maxTasks?: string; + clean?: boolean; + checkout?: boolean; + createIssues?: boolean; + projectId?: string; + goalId?: string; + assigneeAgentId?: string; + issueStatus?: string; + billingCode?: string; + continueOnError?: boolean; +} + +interface BenchValidateOptions extends BaseClientOptions { + tasks: string; + maxTasks?: string; +} + +interface NormalizationDefaults { + maxCostUsd?: number; + maxWallClockMinutes?: number; +} + +interface RunPaperclipBenchOptions { + tasksPath: string; + benchmark: string; + outputDir: string; + runId: string; + paperclipVersion: string; + model: string; + runner: BenchRunner; + maxCostUsd?: number; + maxWallClockMinutes?: number; + maxTasks?: number; + clean?: boolean; + checkout?: boolean; + issueProvisioner?: (task: BenchTask, issueMarkdown: string) => Promise; + runLiveTask?: (input: BenchLiveTaskInput) => Promise; + continueOnError?: boolean; +} + +interface BenchIssueResolution { + issue: Issue; + source: "created" | "reused"; +} + +interface BenchLiveTaskInput { + task: BenchTask; + taskDir: string; + outputDir: string; + issueMarkdown: string; + issue: Issue; + issueSource: "created" | "reused"; + benchmark: string; + runId: string; + startedAt: string; + defaultMaxCostUsd?: number; + defaultMaxWallClockMinutes?: number; +} + +interface BenchLiveTaskOutcome { + issue: Issue; + runIds: string[]; + status: BenchTaskStatus; + patchPath?: string; + failureStage?: string | null; + notes?: string; +} + +class BenchStageError extends Error { + readonly stage: string; + + constructor(stage: string, message: string) { + super(message); + this.stage = stage; + } +} + +const TERMINAL_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]); +const TERMINAL_ISSUE_STATUSES = new Set(["done", "blocked", "cancelled", "in_review"]); +const BENCH_POLL_INTERVAL_MS = 1000; + +export function registerBenchCommands(program: Command): void { + const bench = program + .command("bench") + .description("Prepare and run Paperclip benchmark task suites"); + + addCommonClientOptions( + bench + .command("run") + .description("Normalize a JSONL benchmark suite into Paperclip-ready run artifacts") + .requiredOption("--tasks ", "JSONL task file") + .option("--benchmark ", "Benchmark name", "paperclip_native") + .option("--output ", "Run output directory") + .option("--run-id ", "Stable run id for repeatable output paths") + .option("--paperclip-version ", "Paperclip version label", "current") + .option("--model ", "Model or bundle label", "unspecified") + .option("--runner ", "Runner mode: prepare or paperclip", "prepare") + .option("--max-cost-usd ", "Default per-task cost budget") + .option("--max-wall-clock-minutes ", "Default per-task wall-clock budget") + .option("--max-tasks ", "Limit the number of tasks read from the suite") + .option("--clean", "Remove the output directory before writing") + .option("--checkout", "Clone/check out task repositories into each task workspace") + .option("--create-issues", "Create one Paperclip issue per benchmark task") + .option("--project-id ", "Project ID for created benchmark issues") + .option("--goal-id ", "Goal ID for created benchmark issues") + .option("--assignee-agent-id ", "Assignee agent ID for created benchmark issues") + .option("--issue-status ", "Status for created benchmark issues", "todo") + .option("--billing-code ", "Billing code for created benchmark issues") + .option("--continue-on-error", "Write failed task results and continue") + .action(async (opts: BenchRunOptions) => { + try { + const runner = normalizeBenchRunner(opts.runner); + const benchmark = normalizeIdentifier(opts.benchmark ?? "paperclip_native", "benchmark"); + const runId = opts.runId?.trim() || buildDefaultRunId(benchmark); + const outputDir = opts.output?.trim() || path.join(".tmp", "paperclip-bench", runId); + const maxCostUsd = parseOptionalPositiveNumber(opts.maxCostUsd, "max-cost-usd"); + const maxWallClockMinutes = parseOptionalPositiveNumber( + opts.maxWallClockMinutes, + "max-wall-clock-minutes", + ); + const maxTasks = parseOptionalPositiveInteger(opts.maxTasks, "max-tasks"); + + const issueProvisioner = opts.createIssues || runner === "paperclip" + ? buildIssueProvisioner({ + ...opts, + benchmark, + runId, + }) + : undefined; + const runLiveTask = runner === "paperclip" + ? buildPaperclipLiveTaskRunner({ + ...opts, + benchmark, + runId, + }) + : undefined; + + const manifest = await runPaperclipBench({ + tasksPath: opts.tasks, + benchmark, + outputDir, + runId, + paperclipVersion: opts.paperclipVersion?.trim() || "current", + model: opts.model?.trim() || "unspecified", + runner, + maxCostUsd, + maxWallClockMinutes, + maxTasks, + clean: opts.clean, + checkout: opts.checkout, + issueProvisioner, + runLiveTask, + continueOnError: opts.continueOnError, + }); + + if (opts.json) { + printOutput(manifest, { json: true }); + return; + } + + console.log(pc.bold(runner === "prepare" ? "Paperclip bench run prepared" : "Paperclip bench run completed")); + console.log(`Run: ${manifest.runId}`); + console.log(`Benchmark: ${manifest.benchmark}`); + console.log(`Runner: ${manifest.runner}`); + console.log(`Tasks: ${manifest.taskCount}`); + console.log(`Output: ${manifest.outputDir}`); + if (issueProvisioner) { + const created = manifest.tasks.filter((task) => task.paperclipIssue?.source === "created").length; + const reused = manifest.tasks.filter((task) => task.paperclipIssue?.source === "reused").length; + if (created > 0) console.log(`Issues created: ${created}`); + if (reused > 0) console.log(`Issues reused: ${reused}`); + } + } catch (err) { + handleCommandError(err); + } + }), + { includeCompany: true }, + ); + + addCommonClientOptions( + bench + .command("validate") + .description("Validate a JSONL benchmark task suite") + .requiredOption("--tasks ", "JSONL task file") + .option("--max-tasks ", "Limit the number of tasks read from the suite") + .action(async (opts: BenchValidateOptions) => { + try { + const maxTasks = parseOptionalPositiveInteger(opts.maxTasks, "max-tasks"); + const tasks = await loadBenchTasks(opts.tasks, {}, maxTasks); + + if (opts.json) { + printOutput({ taskCount: tasks.length, taskIds: tasks.map((task) => task.id) }, { json: true }); + return; + } + + console.log(pc.bold("Benchmark suite is valid")); + console.log(`Tasks: ${tasks.length}`); + for (const task of tasks) { + console.log(`- ${task.id}: ${task.title}`); + } + } catch (err) { + handleCommandError(err); + } + }), + ); +} + +export async function runPaperclipBench(options: RunPaperclipBenchOptions): Promise { + const defaults: NormalizationDefaults = { + maxCostUsd: options.maxCostUsd, + maxWallClockMinutes: options.maxWallClockMinutes, + }; + const tasks = await loadBenchTasks(options.tasksPath, defaults, options.maxTasks); + const outputDir = path.resolve(options.outputDir); + + if (options.clean) { + await rm(outputDir, { recursive: true, force: true }); + } + await mkdir(outputDir, { recursive: true }); + + const createdAt = new Date().toISOString(); + const results: BenchResult[] = []; + const manifestTasks: BenchManifestTask[] = []; + + for (const task of tasks) { + const safeId = safePathSegment(task.id); + const taskDir = path.join(outputDir, "tasks", safeId); + await mkdir(taskDir, { recursive: true }); + + const issuePath = path.join(taskDir, "issue.md"); + const taskPath = path.join(taskDir, "task.json"); + const tracePath = path.join(taskDir, "trace.json"); + const workspacePath = options.checkout ? path.join(taskDir, "repo") : undefined; + const issueMarkdown = renderBenchIssueMarkdown(task, { + benchmark: options.benchmark, + runId: options.runId, + model: options.model, + paperclipVersion: options.paperclipVersion, + }); + + const startedAt = new Date().toISOString(); + const events: Array> = [ + { at: startedAt, type: "task.prepared", taskId: task.id }, + ]; + + let paperclipIssue: BenchManifestTask["paperclipIssue"]; + let paperclipRunIds: string[] | undefined; + let patchPath: string | undefined; + let status: BenchResult["status"] = "prepared"; + let failureStage: string | null = null; + let notes: string | undefined; + + try { + await writeFile(issuePath, issueMarkdown, "utf8"); + await writeFile(taskPath, JSON.stringify(task, null, 2), "utf8"); + + if (options.checkout) { + if (!task.repo?.url && !task.repo?.path) { + throw new Error(`Task ${task.id} cannot be checked out because it has no repo.url or repo.path`); + } + checkoutTaskRepo(task, workspacePath!); + events.push({ at: new Date().toISOString(), type: "repo.checked_out", path: workspacePath }); + } + + let resolvedIssue: Issue | undefined; + if (options.issueProvisioner) { + const issueResolution = await options.issueProvisioner(task, issueMarkdown); + resolvedIssue = issueResolution.issue; + paperclipIssue = { + id: issueResolution.issue.id, + identifier: issueResolution.issue.identifier ?? undefined, + status: issueResolution.issue.status ?? undefined, + source: issueResolution.source, + }; + status = issueResolution.source === "created" ? "issue_created" : "issue_reused"; + events.push({ + at: new Date().toISOString(), + type: issueResolution.source === "created" ? "paperclip.issue_created" : "paperclip.issue_reused", + issueId: issueResolution.issue.id, + identifier: issueResolution.issue.identifier, + }); + } + + if (options.runner === "paperclip") { + if (!resolvedIssue) { + throw new BenchStageError( + "issue_provision", + `Runner "${options.runner}" requires an issue provisioner or a task row with an existing issue reference`, + ); + } + if (!options.runLiveTask) { + throw new BenchStageError("paperclip_runner", "Live Paperclip runner is not configured"); + } + + const liveOutcome = await options.runLiveTask({ + task, + taskDir, + outputDir, + issueMarkdown, + issue: resolvedIssue, + issueSource: paperclipIssue?.source ?? "created", + benchmark: options.benchmark, + runId: options.runId, + startedAt, + defaultMaxCostUsd: options.maxCostUsd, + defaultMaxWallClockMinutes: options.maxWallClockMinutes, + }); + paperclipIssue = { + id: liveOutcome.issue.id, + identifier: liveOutcome.issue.identifier ?? undefined, + status: liveOutcome.issue.status ?? undefined, + source: paperclipIssue?.source, + }; + paperclipRunIds = liveOutcome.runIds; + patchPath = liveOutcome.patchPath; + status = liveOutcome.status; + failureStage = liveOutcome.failureStage ?? null; + notes = liveOutcome.notes; + events.push({ + at: new Date().toISOString(), + type: "paperclip.task_completed", + issueId: liveOutcome.issue.id, + issueStatus: liveOutcome.issue.status, + runIds: liveOutcome.runIds, + patchPath: liveOutcome.patchPath ?? null, + failureStage: liveOutcome.failureStage ?? null, + }); + } + } catch (err) { + status = "failed"; + failureStage = + err instanceof BenchStageError + ? err.stage + : options.runner === "paperclip" + ? "paperclip_runner" + : options.issueProvisioner + ? "prepare_or_create_issue" + : "prepare"; + notes = err instanceof Error ? err.message : String(err); + events.push({ + at: new Date().toISOString(), + type: "task.failed", + stage: failureStage, + error: notes, + }); + + if (!options.continueOnError) { + await writeTrace(tracePath, task.id, startedAt, status, events, notes, { + paperclipIssueId: paperclipIssue?.id ?? null, + paperclipIssueIdentifier: paperclipIssue?.identifier ?? null, + paperclipRunIds: paperclipRunIds ?? [], + patchPath: patchPath ?? null, + failureStage, + }); + throw err; + } + } + + const endedAt = new Date().toISOString(); + events.push({ at: endedAt, type: "task.finished", status }); + await writeTrace(tracePath, task.id, startedAt, status, events, notes, { + paperclipIssueId: paperclipIssue?.id ?? null, + paperclipIssueIdentifier: paperclipIssue?.identifier ?? null, + paperclipRunIds: paperclipRunIds ?? [], + patchPath: patchPath ?? null, + failureStage, + }); + + manifestTasks.push({ + id: task.id, + title: task.title, + issuePath: path.relative(outputDir, issuePath), + taskPath: path.relative(outputDir, taskPath), + tracePath: path.relative(outputDir, tracePath), + patchPath, + workspacePath: workspacePath ? path.relative(outputDir, workspacePath) : undefined, + repo: task.repo, + successChecks: task.successChecks, + requiredArtifacts: task.requiredArtifacts, + paperclipRunIds, + status, + paperclipIssue, + }); + + results.push({ + runId: options.runId, + benchmark: options.benchmark, + taskId: task.id, + status, + issuePath: path.relative(outputDir, issuePath), + taskPath: path.relative(outputDir, taskPath), + tracePath: path.relative(outputDir, tracePath), + patchPath, + workspacePath: workspacePath ? path.relative(outputDir, workspacePath) : undefined, + paperclipIssueId: paperclipIssue?.id, + paperclipIssueIdentifier: paperclipIssue?.identifier, + paperclipRunIds, + failureStage, + notes, + }); + } + + const manifest: BenchManifest = { + schemaVersion: 1, + runId: options.runId, + benchmark: options.benchmark, + paperclipVersion: options.paperclipVersion, + model: options.model, + runner: options.runner, + createdAt, + outputDir, + maxCostUsd: options.maxCostUsd, + maxWallClockMinutes: options.maxWallClockMinutes, + taskCount: manifestTasks.length, + tasks: manifestTasks, + }; + + await writeFile(path.join(outputDir, "manifest.json"), JSON.stringify(manifest, null, 2), "utf8"); + await writeFile( + path.join(outputDir, "results.jsonl"), + `${results.map((result) => JSON.stringify(result)).join("\n")}\n`, + "utf8", + ); + + return manifest; +} + +export async function loadBenchTasks( + filePath: string, + defaults: NormalizationDefaults = {}, + maxTasks?: number, +): Promise { + const contents = await readFile(filePath, "utf8"); + const rawTasks = parseJsonlObjects(contents, filePath); + const selected = maxTasks === undefined ? rawTasks : rawTasks.slice(0, maxTasks); + const seen = new Set(); + + return selected.map((raw, index) => { + const task = normalizeBenchTask(raw, { + lineLabel: `${filePath}:${index + 1}`, + defaults, + }); + if (seen.has(task.id)) { + throw new Error(`Duplicate benchmark task id '${task.id}' in ${filePath}`); + } + seen.add(task.id); + return task; + }); +} + +export function normalizeBenchTask( + raw: Record, + opts: { lineLabel: string; defaults?: NormalizationDefaults }, +): BenchTask { + const defaults = opts.defaults ?? {}; + const id = firstString(raw, ["id", "task_id", "taskId"]); + const title = firstString(raw, ["title", "name"]); + const problemStatement = firstString(raw, [ + "problemStatement", + "problem_statement", + "description", + "issue", + "prompt", + "goal", + ]); + + if (!id) throw new Error(`${opts.lineLabel}: benchmark task requires id`); + normalizeIdentifier(id, "task id"); + if (!title) throw new Error(`${opts.lineLabel}: benchmark task '${id}' requires title`); + if (!problemStatement) { + throw new Error(`${opts.lineLabel}: benchmark task '${id}' requires problemStatement, description, issue, prompt, or goal`); + } + + return { + id, + title, + problemStatement, + repo: normalizeRepoSpec(raw), + paperclipIssueRef: normalizeIssueRef(raw), + maxWallClockMinutes: firstNumber(raw, [ + "maxWallClockMinutes", + "max_wall_clock_minutes", + "timeBudgetMinutes", + "time_budget_minutes", + ]) ?? defaults.maxWallClockMinutes, + maxCostUsd: firstNumber(raw, ["maxCostUsd", "max_cost_usd", "tokenBudgetUsd", "token_budget_usd"]) ?? defaults.maxCostUsd, + successChecks: firstStringArray(raw, ["successChecks", "success_checks", "checks"]), + humanRubric: normalizeRubric(raw.humanRubric ?? raw.human_rubric), + requiredArtifacts: firstStringArray(raw, ["requiredArtifacts", "required_artifacts"]), + metadata: asRecord(raw.metadata), + }; +} + +export function renderBenchIssueMarkdown( + task: BenchTask, + run: { + benchmark: string; + runId: string; + model: string; + paperclipVersion: string; + }, +): string { + const lines: string[] = [ + `# ${task.title}`, + "", + "## Benchmark Context", + "", + `- Task ID: \`${task.id}\``, + `- Benchmark: \`${run.benchmark}\``, + `- Run ID: \`${run.runId}\``, + `- Paperclip version: \`${run.paperclipVersion}\``, + `- Model or bundle: \`${run.model}\``, + ]; + + if (task.maxWallClockMinutes !== undefined || task.maxCostUsd !== undefined) { + lines.push("", "## Budgets", ""); + if (task.maxWallClockMinutes !== undefined) { + lines.push(`- Max wall-clock minutes: ${task.maxWallClockMinutes}`); + } + if (task.maxCostUsd !== undefined) { + lines.push(`- Max cost USD: ${task.maxCostUsd}`); + } + } + + if (task.repo) { + lines.push("", "## Repository", ""); + if (task.repo.url) lines.push(`- URL: ${task.repo.url}`); + if (task.repo.path) lines.push(`- Path: ${task.repo.path}`); + if (task.repo.baseCommit) lines.push(`- Base commit: ${task.repo.baseCommit}`); + } + + lines.push( + "", + "## Task", + "", + task.problemStatement.trim(), + "", + "## Rules", + "", + "- Make the minimal correct code change.", + "- Add or update tests when appropriate.", + "- Run the smallest relevant verification that proves the change.", + "- Do not modify unrelated files.", + "- At the end, produce a final summary with changes, verification, and remaining risks.", + ); + + if (task.successChecks.length > 0) { + lines.push("", "## Success Checks", ""); + lines.push(...task.successChecks.map((check) => `- ${check}`)); + } + + if (task.requiredArtifacts.length > 0) { + lines.push("", "## Required Artifacts", ""); + lines.push(...task.requiredArtifacts.map((artifact) => `- ${artifact}`)); + } + + if (task.humanRubric.length > 0) { + lines.push("", "## Human Rubric", ""); + for (const item of task.humanRubric) { + const suffix = item.maxScore === undefined ? "" : ` (${item.maxScore} pts)`; + const description = item.description ? `: ${item.description}` : ""; + lines.push(`- ${item.name}${suffix}${description}`); + } + } + + return `${lines.join("\n")}\n`; +} + +export function normalizeBenchRunner(value: string | undefined): BenchRunner { + const normalized = value?.trim() || "prepare"; + if (normalized === "prepare" || normalized === "paperclip") { + return normalized; + } + throw new Error(`Unsupported bench runner '${normalized}'. Expected one of: prepare, paperclip.`); +} + +function buildIssueProvisioner(opts: BenchRunOptions & { benchmark: string; runId: string }) { + const ctx = resolveCommandContext(opts); + return async (task: BenchTask, issueMarkdown: string): Promise => { + const issueRef = task.paperclipIssueRef?.trim(); + if (issueRef) { + const existing = await ctx.api.get(`/api/issues/${encodeURIComponent(issueRef)}`); + if (!existing) { + throw new BenchStageError("issue_provision", `Benchmark task ${task.id} references missing issue ${issueRef}`); + } + return { issue: existing, source: "reused" }; + } + + if (!ctx.companyId) { + throw new BenchStageError( + "issue_provision", + "Company ID is required to create benchmark issues. Pass --company-id, set PAPERCLIP_COMPANY_ID, or reuse an existing issue from the task row.", + ); + } + + const payload = { + title: `[${opts.benchmark}] ${task.title}`, + description: issueMarkdown, + status: opts.issueStatus ?? "todo", + priority: "medium", + assigneeAgentId: opts.assigneeAgentId, + projectId: opts.projectId, + goalId: opts.goalId, + billingCode: opts.billingCode || `bench:${opts.runId}`, + }; + const issue = await ctx.api.post(`/api/companies/${ctx.companyId}/issues`, payload); + if (!issue) { + throw new BenchStageError("issue_provision", `Paperclip API did not return the created issue for task ${task.id}`); + } + return { issue, source: "created" }; + }; +} + +function buildPaperclipLiveTaskRunner(opts: BenchRunOptions & { benchmark: string; runId: string }) { + const ctx = resolveCommandContext(opts); + + return async (input: BenchLiveTaskInput): Promise => { + const assigneeAgentId = input.issue.assigneeAgentId ?? (opts.assigneeAgentId?.trim() || null); + if (!assigneeAgentId) { + throw new BenchStageError( + "issue_checkout", + `Benchmark issue ${input.issue.identifier ?? input.issue.id} has no assigneeAgentId to run`, + ); + } + + try { + await ctx.api.post(`/api/issues/${input.issue.id}/checkout`, { + agentId: assigneeAgentId, + expectedStatuses: ["backlog", "todo", "blocked", "in_review", "in_progress"], + }); + } catch (err) { + if (err instanceof ApiRequestError) { + throw new BenchStageError("issue_checkout", err.message); + } + throw err; + } + + const wakeResponse = await ctx.api.post(`/api/agents/${assigneeAgentId}/wakeup`, { + source: "assignment", + triggerDetail: "manual", + reason: "bench_runner", + payload: { + benchmark: input.benchmark, + runId: input.runId, + taskId: input.task.id, + issueId: input.issue.id, + }, + idempotencyKey: `bench:${input.runId}:${input.task.id}:${input.issue.id}`, + }); + + const observedRunIds = new Set(); + const initialRunId = readWakeupRunId(wakeResponse); + if (initialRunId) observedRunIds.add(initialRunId); + + const budgetMs = resolveWallClockBudgetMs(input.task, input.defaultMaxWallClockMinutes); + const deadline = budgetMs ? Date.now() + budgetMs : null; + let latestIssue = await ctx.api.get(`/api/issues/${input.issue.id}`); + if (!latestIssue) { + throw new BenchStageError("issue_poll", `Benchmark issue ${input.issue.id} disappeared while polling`); + } + + let finalRun: HeartbeatRun | null = null; + let failureStage: string | null = null; + let notes = readWakeupNotes(wakeResponse); + + while (true) { + const activeRunSummary = await ctx.api.get<{ id: string } | null>(`/api/issues/${input.issue.id}/active-run`); + const activeRunId = typeof activeRunSummary?.id === "string" ? activeRunSummary.id : null; + if (activeRunId) { + observedRunIds.add(activeRunId); + const run = await ctx.api.get(`/api/heartbeat-runs/${activeRunId}`); + if (run) { + finalRun = run; + const costUsd = readHeartbeatRunCostUsd(run); + const maxCostUsd = input.task.maxCostUsd ?? input.defaultMaxCostUsd; + if ( + maxCostUsd !== undefined + && costUsd !== null + && costUsd > maxCostUsd + ) { + try { + await ctx.api.post(`/api/heartbeat-runs/${run.id}/cancel`); + } catch (cancelErr) { + notes = appendNote( + notes, + `Cost budget exceeded at ${costUsd.toFixed(2)} USD, and cancelling run ${run.id} failed: ${cancelErr instanceof Error ? cancelErr.message : String(cancelErr)}`, + ); + } + failureStage = "cost_budget_exceeded"; + notes = appendNote(notes, `Cancelled run ${run.id} after exceeding max cost budget ${maxCostUsd} USD.`); + latestIssue = (await ctx.api.get(`/api/issues/${input.issue.id}`)) ?? latestIssue; + break; + } + } + } + + latestIssue = (await ctx.api.get(`/api/issues/${input.issue.id}`)) ?? latestIssue; + if (latestIssue.executionRunId) observedRunIds.add(latestIssue.executionRunId); + + if (TERMINAL_ISSUE_STATUSES.has(latestIssue.status)) { + if (latestIssue.executionRunId) { + finalRun = await ctx.api.get(`/api/heartbeat-runs/${latestIssue.executionRunId}`, { ignoreNotFound: true }); + } + break; + } + + if (!activeRunId && latestIssue.executionRunId) { + finalRun = await ctx.api.get(`/api/heartbeat-runs/${latestIssue.executionRunId}`, { ignoreNotFound: true }); + if (finalRun?.status && TERMINAL_RUN_STATUSES.has(finalRun.status)) { + break; + } + } + + if (deadline && Date.now() >= deadline) { + failureStage = failureStage ?? "run_timeout"; + if (activeRunId) { + try { + await ctx.api.post(`/api/heartbeat-runs/${activeRunId}/cancel`); + notes = appendNote(notes, `Cancelled run ${activeRunId} after wall-clock budget expired.`); + } catch (cancelErr) { + notes = appendNote( + notes, + `Wall-clock budget expired for run ${activeRunId}, and cancelling it failed: ${cancelErr instanceof Error ? cancelErr.message : String(cancelErr)}`, + ); + } + } + latestIssue = (await ctx.api.get(`/api/issues/${input.issue.id}`)) ?? latestIssue; + break; + } + + await delay(BENCH_POLL_INTERVAL_MS); + } + + const issueRuns = await listIssueRunIdsSince(ctx, latestIssue.companyId, assigneeAgentId, latestIssue.id, input.startedAt); + for (const runId of issueRuns) observedRunIds.add(runId); + + const sortedRunIds = [...observedRunIds]; + if (!failureStage && latestIssue.status !== "done") { + failureStage = mapIssueStatusToFailureStage(latestIssue.status, finalRun?.status ?? null); + } + + let extractedPatchPath: string | undefined; + try { + extractedPatchPath = await extractIssuePatch({ + issue: latestIssue, + outputDir: input.outputDir, + taskDir: input.taskDir, + }); + if (latestIssue.currentExecutionWorkspace?.mode === "shared_workspace") { + notes = appendNote( + notes, + `Patch was extracted from shared workspace ${latestIssue.currentExecutionWorkspace.name}; unrelated dirty changes in that workspace can contaminate the diff.`, + ); + } + if (!extractedPatchPath) { + failureStage = failureStage ?? "patch_missing"; + notes = appendNote(notes, `No git diff was available in the execution workspace for issue ${latestIssue.identifier ?? latestIssue.id}.`); + } + } catch (err) { + failureStage = failureStage ?? "patch_extract"; + notes = appendNote(notes, err instanceof Error ? err.message : String(err)); + } + + return { + issue: latestIssue, + runIds: sortedRunIds, + status: latestIssue.status, + patchPath: extractedPatchPath, + failureStage, + notes, + }; + }; +} + +function parseJsonlObjects(contents: string, filePath: string): Array> { + const rows: Array> = []; + const lines = contents.split(/\r?\n/); + + lines.forEach((line, index) => { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) return; + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`${filePath}:${index + 1}: invalid JSONL row: ${message}`); + } + + if (!isRecord(parsed)) { + throw new Error(`${filePath}:${index + 1}: expected a JSON object`); + } + rows.push(parsed); + }); + + if (rows.length === 0) { + throw new Error(`${filePath}: no benchmark tasks found`); + } + + return rows; +} + +function normalizeRepoSpec(raw: Record): BenchRepoSpec | undefined { + const rawRepo = raw.repo; + const startingCommit = firstString(raw, ["startingCommit", "starting_commit", "baseCommit", "base_commit"]); + + if (typeof rawRepo === "string" && rawRepo.trim()) { + return compactRepoSpec({ url: rawRepo.trim(), baseCommit: startingCommit }); + } + + if (isRecord(rawRepo)) { + return compactRepoSpec({ + url: firstString(rawRepo, ["url", "remote", "repository"]), + path: firstString(rawRepo, ["path", "localPath", "local_path"]), + baseCommit: firstString(rawRepo, ["baseCommit", "base_commit", "startingCommit", "starting_commit"]) ?? startingCommit, + }); + } + + return startingCommit ? { baseCommit: startingCommit } : undefined; +} + +function normalizeIssueRef(raw: Record): string | undefined { + const metadata = asRecord(raw.metadata); + return firstString(raw, [ + "paperclipIssueRef", + "paperclip_issue_ref", + "paperclipIssueId", + "paperclip_issue_id", + "paperclipIssueIdentifier", + "paperclip_issue_identifier", + "existingIssueId", + "existing_issue_id", + "issueRef", + "issue_ref", + ]) ?? (metadata + ? firstString(metadata, [ + "paperclipIssueRef", + "paperclipIssueId", + "paperclipIssueIdentifier", + "existingIssueId", + "issueRef", + ]) + : undefined); +} + +function compactRepoSpec(repo: BenchRepoSpec): BenchRepoSpec | undefined { + const compact = { + url: repo.url?.trim() || undefined, + path: repo.path?.trim() || undefined, + baseCommit: repo.baseCommit?.trim() || undefined, + }; + return compact.url || compact.path || compact.baseCommit ? compact : undefined; +} + +function normalizeRubric(value: unknown): BenchRubricItem[] { + if (Array.isArray(value)) { + return value.map((item, index) => { + if (typeof item === "string" && item.trim()) { + return { name: item.trim() }; + } + if (isRecord(item)) { + const name = firstString(item, ["name", "criterion", "label"]); + if (!name) throw new Error(`human_rubric[${index}] requires name`); + return { + name, + description: firstString(item, ["description", "notes"]), + maxScore: firstNumber(item, ["maxScore", "max_score", "points"]), + }; + } + throw new Error(`human_rubric[${index}] must be a string or object`); + }); + } + + if (isRecord(value)) { + return Object.entries(value).map(([name, raw]) => { + if (typeof raw === "number") return { name, maxScore: raw }; + if (typeof raw === "string") return { name, description: raw }; + if (isRecord(raw)) { + return { + name, + description: firstString(raw, ["description", "notes"]), + maxScore: firstNumber(raw, ["maxScore", "max_score", "points"]), + }; + } + return { name }; + }); + } + + return []; +} + +function checkoutTaskRepo(task: BenchTask, workspacePath: string): void { + const source = task.repo?.url ?? task.repo?.path; + if (!source) throw new Error(`Task ${task.id} has no repository source`); + + const clone = spawnSync("git", ["clone", "--no-tags", source, workspacePath], { + encoding: "utf8", + }); + if (clone.status !== 0) { + throw new Error(`git clone failed for task ${task.id}: ${clone.stderr || clone.stdout}`); + } + + if (task.repo?.baseCommit) { + const checkout = spawnSync("git", ["checkout", task.repo.baseCommit], { + cwd: workspacePath, + encoding: "utf8", + }); + if (checkout.status !== 0) { + throw new Error(`git checkout failed for task ${task.id}: ${checkout.stderr || checkout.stdout}`); + } + } +} + +async function writeTrace( + tracePath: string, + taskId: string, + startedAt: string, + status: BenchResult["status"], + events: Array>, + error?: string, + extra?: Record, +): Promise { + await writeFile( + tracePath, + JSON.stringify( + { + schemaVersion: 1, + taskId, + startedAt, + endedAt: new Date().toISOString(), + status, + error, + events, + ...(extra ?? {}), + }, + null, + 2, + ), + "utf8", + ); +} + +function readWakeupRunId(response: AgentWakeupResponse | null): string | null { + if (!response) return null; + if ("id" in response && typeof response.id === "string") { + return response.id; + } + if ("executionRunId" in response && typeof response.executionRunId === "string") { + return response.executionRunId; + } + return null; +} + +function readWakeupNotes(response: AgentWakeupResponse | null): string | undefined { + if (!response || !("status" in response) || response.status !== "skipped") return undefined; + const parts = [response.reason, response.message].filter((value): value is string => Boolean(value && value.trim())); + return parts.length > 0 ? `Heartbeat wakeup skipped: ${parts.join(" - ")}` : undefined; +} + +function resolveWallClockBudgetMs(task: BenchTask, defaultMaxWallClockMinutes?: number): number | undefined { + const minutes = task.maxWallClockMinutes ?? defaultMaxWallClockMinutes; + if (minutes === undefined) return undefined; + return Math.max(1, Math.ceil(minutes * 60_000)); +} + +function readHeartbeatRunCostUsd(run: HeartbeatRun | null | undefined): number | null { + const usage = asRecord(run?.usageJson); + const result = asRecord(run?.resultJson); + return firstFiniteNumber( + usage ? [usage.costUsd, usage.cost_usd, usage.total_cost_usd] : [], + result ? [result.costUsd, result.cost_usd, result.total_cost_usd] : [], + ); +} + +async function listIssueRunIdsSince( + ctx: ReturnType, + companyId: string, + agentId: string, + issueId: string, + startedAt: string, +): Promise { + const runs = await ctx.api.get(`/api/companies/${companyId}/heartbeat-runs?agentId=${agentId}&limit=200`); + if (!Array.isArray(runs)) return []; + const startedAtMs = toTimestamp(startedAt) ?? 0; + return runs + .filter((run) => { + const context = asRecord(run.contextSnapshot); + return ( + typeof run.id === "string" + && (toTimestamp(run.createdAt) ?? 0) >= startedAtMs + && context?.issueId === issueId + ); + }) + .sort((left, right) => (toTimestamp(left.createdAt) ?? 0) - (toTimestamp(right.createdAt) ?? 0)) + .map((run) => run.id); +} + +async function extractIssuePatch(input: { + issue: Issue; + outputDir: string; + taskDir: string; +}): Promise { + const workspaceCwd = input.issue.currentExecutionWorkspace?.cwd; + if (!workspaceCwd) { + throw new BenchStageError( + "patch_extract", + `Issue ${input.issue.identifier ?? input.issue.id} does not have a local execution workspace available for patch extraction`, + ); + } + + const statusBefore = spawnSync("git", ["-C", workspaceCwd, "status", "--porcelain=v1"], { + encoding: "utf8", + }); + if (statusBefore.status !== 0) { + throw new BenchStageError( + "patch_extract", + `git status failed for ${workspaceCwd}: ${statusBefore.stderr || statusBefore.stdout}`, + ); + } + + const untrackedFiles = statusBefore.stdout + .split(/\r?\n/) + .filter((line) => line.startsWith("?? ")) + .map((line) => line.slice(3).trim()) + .filter(Boolean); + if (untrackedFiles.length > 0) { + const addIntent = spawnSync("git", ["-C", workspaceCwd, "add", "-N", "--", ...untrackedFiles], { + encoding: "utf8", + }); + if (addIntent.status !== 0) { + throw new BenchStageError( + "patch_extract", + `git add -N failed for ${workspaceCwd}: ${addIntent.stderr || addIntent.stdout}`, + ); + } + } + + const diff = spawnSync("git", ["-C", workspaceCwd, "diff", "--binary", "--no-ext-diff", "--submodule=diff", "HEAD", "--"], { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + if (diff.status !== 0) { + throw new BenchStageError( + "patch_extract", + `git diff failed for ${workspaceCwd}: ${diff.stderr || diff.stdout}`, + ); + } + if (!diff.stdout.trim()) return undefined; + + const patchFilePath = path.join(input.taskDir, "patch.diff"); + await writeFile(patchFilePath, diff.stdout, "utf8"); + return path.relative(input.outputDir, patchFilePath); +} + +function mapIssueStatusToFailureStage( + status: Issue["status"], + runStatus: HeartbeatRun["status"] | null, +): string | null { + if (status === "done") return null; + if (status === "blocked") return "issue_blocked"; + if (status === "cancelled") return "issue_cancelled"; + if (status === "in_review") return "issue_in_review"; + if (runStatus === "timed_out") return "run_timeout"; + if (runStatus === "cancelled") return "run_cancelled"; + if (runStatus === "failed") return "run_failed"; + return "issue_incomplete"; +} + +function appendNote(existing: string | undefined, addition: string): string { + const trimmed = addition.trim(); + if (!trimmed) return existing ?? ""; + if (!existing?.trim()) return trimmed; + return `${existing}\n${trimmed}`; +} + +function toTimestamp(value: Date | string | null | undefined): number | null { + if (!value) return null; + const timestamp = value instanceof Date ? value.getTime() : new Date(value).getTime(); + return Number.isFinite(timestamp) ? timestamp : null; +} + +function firstFiniteNumber(...groups: unknown[][]): number | null { + for (const group of groups) { + for (const value of group) { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim()) { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + } + } + return null; +} + +function firstString(record: Record, keys: string[]): string | undefined { + for (const key of keys) { + const value = record[key]; + if (typeof value === "string" && value.trim()) return value.trim(); + } + return undefined; +} + +function firstNumber(record: Record, keys: string[]): number | undefined { + for (const key of keys) { + const value = record[key]; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim()) { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + } + return undefined; +} + +function firstStringArray(record: Record, keys: string[]): string[] { + for (const key of keys) { + const value = record[key]; + if (Array.isArray(value)) { + return value + .map((item) => typeof item === "string" ? item.trim() : "") + .filter(Boolean); + } + } + return []; +} + +function asRecord(value: unknown): Record | undefined { + return isRecord(value) ? value : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseOptionalPositiveNumber(value: string | undefined, label: string): number | undefined { + if (value === undefined || value.trim() === "") return undefined; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`--${label} must be a positive number`); + } + return parsed; +} + +function parseOptionalPositiveInteger(value: string | undefined, label: string): number | undefined { + const parsed = parseOptionalPositiveNumber(value, label); + if (parsed === undefined) return undefined; + if (!Number.isInteger(parsed)) { + throw new Error(`--${label} must be a positive integer`); + } + return parsed; +} + +function normalizeIdentifier(value: string, label: string): string { + const trimmed = value.trim(); + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(trimmed)) { + throw new Error(`${label} must start with an alphanumeric character and contain only letters, numbers, '.', '_', ':', or '-'`); + } + return trimmed; +} + +function safePathSegment(value: string): string { + return value.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "") || "task"; +} + +function buildDefaultRunId(benchmark: string): string { + const stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z"); + return `${stamp}-${safePathSegment(benchmark)}`; +} diff --git a/cli/src/index.ts b/cli/src/index.ts index bbec356f342..53867bebccb 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -18,6 +18,7 @@ import { registerActivityCommands } from "./commands/client/activity.js"; import { registerDashboardCommands } from "./commands/client/dashboard.js"; import { registerRoutineCommands } from "./commands/routines.js"; import { registerFeedbackCommands } from "./commands/client/feedback.js"; +import { registerBenchCommands } from "./commands/bench.js"; import { applyDataDirOverride, type DataDirOptionLike } from "./config/data-dir.js"; import { loadPaperclipEnvFile } from "./config/env.js"; import { initTelemetryFromConfigFile, flushTelemetry } from "./telemetry.js"; @@ -147,6 +148,7 @@ registerActivityCommands(program); registerDashboardCommands(program); registerRoutineCommands(program); registerFeedbackCommands(program); +registerBenchCommands(program); registerWorktreeCommands(program); registerEnvLabCommands(program); registerPluginCommands(program); diff --git a/evals/README.md b/evals/README.md index 5974d98cfe1..efc183973ce 100644 --- a/evals/README.md +++ b/evals/README.md @@ -34,6 +34,53 @@ promptfoo eval promptfoo view ``` +## Paperclip Bench + +`paperclipai bench` is the first-party benchmark harness for Paperclip-native and SWE-style task suites. It supports a deterministic `prepare` runner for prompt/render artifacts and a live `paperclip` runner that creates or reuses benchmark issues, wakes the assignee, and captures run metadata plus a final patch artifact. + +```bash +# Prepare the bundled smoke suite +pnpm bench:smoke + +# Validate any JSONL suite +pnpm paperclipai bench validate --tasks evals/bench/tasks/paperclip-native-smoke.jsonl + +# Prepare artifacts for a custom suite +pnpm paperclipai bench run \ + --tasks path/to/tasks.jsonl \ + --benchmark missionbench \ + --paperclip-version current \ + --model anthropic/claude-sonnet-4.5 \ + --max-cost-usd 10 \ + --max-wall-clock-minutes 120 \ + --output .tmp/paperclip-bench/missionbench-run + +# Run tasks live against a Paperclip dev server +pnpm paperclipai bench run \ + --runner paperclip \ + --tasks path/to/tasks.jsonl \ + --benchmark missionbench \ + --paperclip-version current \ + --model anthropic/claude-sonnet-4.5 \ + --company-id \ + --assignee-agent-id \ + --project-id \ + --goal-id \ + --max-cost-usd 10 \ + --max-wall-clock-minutes 120 \ + --output .tmp/paperclip-bench/missionbench-live +``` + +Task rows are JSONL objects. The parser accepts both camelCase and snake_case fields so suites can use MissionBench-style records: + +```json +{"id":"missionbench-001","title":"Add audit logs","repo":"github.com/paperclipai/missionbench-saas","starting_commit":"abc123","problem_statement":"Add organization-level audit logs.","success_checks":["npm test"],"required_artifacts":["implementation patch","final PR summary"]} +``` + +Set `paperclip_issue_id`, `paperclip_issue_identifier`, `paperclipIssueRef`, or the same keys inside `metadata` to reuse an existing benchmark issue from the task row. Otherwise, `--create-issues -C ` creates one Paperclip issue per task for the `prepare` runner, and the live `paperclip` runner creates issues automatically when the task row does not already reference one. Created issues use the normalized prompt as the issue description and set `billingCode` to `bench:` unless `--billing-code` is provided. + +Live-run `results.jsonl` and per-task `trace.json` include the Paperclip issue id, observed heartbeat run ids, the final task status, a relative `patchPath` when a git diff was extracted from the execution workspace, and a `failureStage` when the task stops short of a clean `done` result. + ### What's tested Phase 0 covers narrow behavior evals for the Paperclip heartbeat skill: diff --git a/evals/bench/.gitignore b/evals/bench/.gitignore new file mode 100644 index 00000000000..d3189f1617a --- /dev/null +++ b/evals/bench/.gitignore @@ -0,0 +1,2 @@ +runs/* +!runs/.gitkeep diff --git a/evals/bench/README.md b/evals/bench/README.md new file mode 100644 index 00000000000..67330eb5c7d --- /dev/null +++ b/evals/bench/README.md @@ -0,0 +1,50 @@ +# paperclip-bench + +`paperclip-bench` is the scaffold for Paperclip's benchmark harness and benchmark content. Phase 1 keeps it inside the monorepo under `evals/bench/`, but the layout is intentionally shaped so it can move into a standalone repository later with minimal churn. + +This phase does not implement the runner. It establishes: + +- where imported benchmark manifests belong +- where Paperclip-native tasks belong +- where starter company packages belong +- where runner, evaluator, Python bridge, ops, and generated run artifacts belong +- the initial schema contract for task and suite manifests + +## Layout + +| Path | Purpose | +|---|---| +| `bench.schema.json` | Shared JSON Schema definitions for `paperclip-bench/task/v1` and `paperclip-bench/suite/v1`. | +| `suites/` | Versioned suite manifests such as local smoke, SWE-bench Lite smoke, and Paperclip-native smoke. | +| `benchmarks/imported/` | Imported benchmark metadata and normalized task manifests without vendoring third-party source repositories. | +| `benchmarks/paperclip-native/` | Paperclip-native tasks, prompts, rubrics, evaluator config, and hidden-test placeholders. | +| `companies/` | Starter company packages used to seed Paperclip-native benchmark runs. | +| `fixtures/` | Fixture repo placeholders plus shared env/secrets examples. | +| `src/` | Future TypeScript CLI, task normalization, runners, evaluators, metrics, and reports. | +| `python/` | Future bridge scripts for ecosystems that already expect Python entrypoints. | +| `ops/` | Local Docker and single-machine cloud execution scaffolding. | +| `runs/` | Generated benchmark outputs. Raw runs stay untracked by default. | + +## Manifest Validation + +Phase 1 picks JSON Schema plus YAML manifests. The validation script loads YAML, validates against `bench.schema.json`, and checks both suite and task examples: + +```bash +node evals/bench/scripts/validate-manifests.mjs +# or +pnpm --dir evals/bench run validate:manifests +``` + +## V0 Verification Intent + +The first vertical slice will be considered wired correctly when these commands are all true: + +1. `node evals/bench/scripts/validate-manifests.mjs` passes. +2. A contributor can locate imported tasks, native tasks, starter companies, runner code, Python bridge code, ops files, and generated runs from this directory layout alone. +3. Later phases can add implementation in place without changing manifest ids or top-level directory meaning. + +## Notes + +- `.paperclip.yaml` starter-company files are placeholders in this phase. Their exact import/runtime contract lands with the Paperclip runner integration work. +- Hidden tests are represented as directories only. Real evaluator wiring lands in later phases. +- The current Paperclip CLI already has early bench commands under `cli/src/commands/bench.ts`; this scaffold defines the repository contract those commands can grow into. diff --git a/evals/bench/bench.schema.json b/evals/bench/bench.schema.json new file mode 100644 index 00000000000..a5c21be75e8 --- /dev/null +++ b/evals/bench/bench.schema.json @@ -0,0 +1,272 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://paperclip.ai/schemas/paperclip-bench/bench.schema.json", + "title": "paperclip-bench manifest schema", + "$defs": { + "pathRef": { + "type": "string", + "minLength": 1 + }, + "runnerKind": { + "type": "string", + "enum": ["paperclip", "direct-agent", "noop", "prepare"] + }, + "repo": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "url", "ref"], + "properties": { + "kind": { + "type": "string", + "enum": ["git", "local-fixture"] + }, + "url": { + "type": "string", + "minLength": 1 + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "workdir": { + "type": "string", + "minLength": 1 + } + } + }, + "budget": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_wall_clock_minutes": { + "type": "integer", + "minimum": 1 + }, + "max_cost_usd": { + "type": "number", + "minimum": 0 + }, + "max_tool_calls": { + "type": "integer", + "minimum": 1 + }, + "max_test_runs": { + "type": "integer", + "minimum": 1 + } + } + }, + "stop": { + "type": "object", + "additionalProperties": false, + "properties": { + "stable_idle_minutes": { + "type": "integer", + "minimum": 0 + }, + "require_terminal_issue_state": { + "type": "boolean" + }, + "allow_human_intervention": { + "type": "boolean" + } + } + }, + "runner": { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { + "$ref": "#/$defs/runnerKind" + }, + "mode": { + "type": "string", + "minLength": 1 + } + } + }, + "evaluator": { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { + "type": "string", + "enum": ["command", "swebench", "swebench-pro", "paperclip-native"] + }, + "manifest": { + "$ref": "#/$defs/pathRef" + }, + "prediction_format": { + "type": "string", + "minLength": 1 + }, + "visible_commands": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "hidden_commands": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + } + }, + "taskV1": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "id", "family", "summary", "repo", "runner", "evaluator"], + "properties": { + "$schema": { + "type": "string", + "minLength": 1 + }, + "schema": { + "type": "string", + "const": "paperclip-bench/task/v1" + }, + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]*$" + }, + "family": { + "type": "string", + "minLength": 1 + }, + "summary": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "source": { + "type": "object" + }, + "repo": { + "$ref": "#/$defs/repo" + }, + "prompt": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "$ref": "#/$defs/pathRef" + } + } + }, + "starter_company": { + "type": "object", + "additionalProperties": false, + "properties": { + "package": { + "$ref": "#/$defs/pathRef" + } + } + }, + "runner": { + "$ref": "#/$defs/runner" + }, + "budget": { + "$ref": "#/$defs/budget" + }, + "stop": { + "$ref": "#/$defs/stop" + }, + "evaluator": { + "$ref": "#/$defs/evaluator" + }, + "required_artifacts": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "metrics_profile": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + } + }, + "suiteV1": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "id", "title", "tasks"], + "properties": { + "$schema": { + "type": "string", + "minLength": 1 + }, + "schema": { + "type": "string", + "const": "paperclip-bench/suite/v1" + }, + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "runner": { + "$ref": "#/$defs/runner" + }, + "paperclip_version": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "tasks": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/pathRef" + } + }, + "defaults": { + "type": "object", + "additionalProperties": false, + "properties": { + "budget": { + "$ref": "#/$defs/budget" + }, + "stop": { + "$ref": "#/$defs/stop" + } + } + }, + "notes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + } + } + } +} diff --git a/evals/bench/benchmarks/imported/swebench-lite/README.md b/evals/bench/benchmarks/imported/swebench-lite/README.md new file mode 100644 index 00000000000..dff75435a51 --- /dev/null +++ b/evals/bench/benchmarks/imported/swebench-lite/README.md @@ -0,0 +1,5 @@ +# SWE-bench Lite Imports + +This directory stores normalized task manifests and dataset notes for imported SWE-bench Lite tasks. It does not vendor upstream benchmark repositories or evaluator code. + +Use `tasks/` for one manifest per selected instance. Keep raw upstream metadata under `source` so later bridge code can emit official prediction formats without guessing. diff --git a/evals/bench/benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml b/evals/bench/benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml new file mode 100644 index 00000000000..84571435dde --- /dev/null +++ b/evals/bench/benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml @@ -0,0 +1,39 @@ +$schema: ../../../../bench.schema.json#/$defs/taskV1 +schema: paperclip-bench/task/v1 +id: swebench-lite:pytest-dev__pytest-6116 +family: swebench-lite +summary: Fix a small pytest CLI behavior regression imported from SWE-bench Lite. +description: Imported smoke task used to validate the scaffold and preserve upstream task metadata. +source: + dataset: SWE-bench/SWE-bench_Lite + instance_id: pytest-dev__pytest-6116 + repo: pytest-dev/pytest + base_commit: 17f7b44f2f1b8b1a6b2cb67f7e6f01a8a5b9ea12 + version: "6.0" +repo: + kind: git + url: https://github.com/pytest-dev/pytest.git + ref: 17f7b44f2f1b8b1a6b2cb67f7e6f01a8a5b9ea12 +runner: + kind: prepare + mode: isolated-instance +budget: + max_wall_clock_minutes: 60 + max_cost_usd: 5 + max_tool_calls: 150 +stop: + stable_idle_minutes: 5 + require_terminal_issue_state: true +evaluator: + kind: swebench + prediction_format: swebench-jsonl +required_artifacts: + - prediction.patch + - results.json +metrics_profile: + - patch + - cost + - evaluator +tags: + - imported + - smoke diff --git a/evals/bench/benchmarks/paperclip-native/README.md b/evals/bench/benchmarks/paperclip-native/README.md new file mode 100644 index 00000000000..7d0bb9d2f91 --- /dev/null +++ b/evals/bench/benchmarks/paperclip-native/README.md @@ -0,0 +1,3 @@ +# Paperclip-Native Tasks + +Paperclip-native tasks test the control plane itself: decomposition, blockers, handoffs, artifact quality, liveness, and reviewability. Each task owns its prompt, rubric, evaluator description, and hidden-test placeholders in one directory. diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/evaluator.yaml b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/evaluator.yaml new file mode 100644 index 00000000000..ea0d6cb3fbd --- /dev/null +++ b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/evaluator.yaml @@ -0,0 +1,8 @@ +kind: command +visible_commands: + - pnpm test + - pnpm -r typecheck +hidden_commands: + - pnpm test:e2e:audit-log +notes: + - Hidden tests stay outside the agent workspace once evaluator wiring lands. diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/hidden-tests/README.md b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/hidden-tests/README.md new file mode 100644 index 00000000000..fcfc738a2b3 --- /dev/null +++ b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/hidden-tests/README.md @@ -0,0 +1,3 @@ +# Hidden Tests Placeholder + +Real hidden tests are intentionally omitted in Phase 1. This directory exists so later phases have a stable location for evaluator-only assets. diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/issue.md b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/issue.md new file mode 100644 index 00000000000..1cd08b7c443 --- /dev/null +++ b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/issue.md @@ -0,0 +1,10 @@ +# Add Organization Audit Logs + +Implement organization-level audit logs for account settings changes in the demo SaaS fixture. + +Definition of done: + +- account settings mutations create durable audit-log records +- the app exposes enough UI or API surface for operators to inspect the new entries +- relevant tests are updated or added +- the final summary names what changed, what was verified, and remaining risks diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/rubric.md b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/rubric.md new file mode 100644 index 00000000000..1dad3f60120 --- /dev/null +++ b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/rubric.md @@ -0,0 +1,9 @@ +# Rubric + +Score from 0-5 on: + +- architecture +- minimality +- test quality +- final summary quality +- operational reviewability diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml new file mode 100644 index 00000000000..8ca3f75e57d --- /dev/null +++ b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml @@ -0,0 +1,49 @@ +$schema: ../../../../bench.schema.json#/$defs/taskV1 +schema: paperclip-bench/task/v1 +id: pclip-native-001-audit-log-mission +family: paperclip-native +summary: Add organization-level audit logs to a demo SaaS app through a small engineering squad. +description: Native smoke task focused on implementation, verification, and final review artifacts. +repo: + kind: local-fixture + url: ../../../../fixtures/repos/demo-saas + ref: scaffold + workdir: . +prompt: + path: ./issue.md +starter_company: + package: ../../../../companies/eng-squad-basic +runner: + kind: paperclip + mode: isolated-instance +budget: + max_wall_clock_minutes: 120 + max_cost_usd: 10 + max_tool_calls: 300 + max_test_runs: 20 +stop: + stable_idle_minutes: 5 + require_terminal_issue_state: true + allow_human_intervention: false +evaluator: + kind: command + manifest: ./evaluator.yaml + visible_commands: + - pnpm test + - pnpm -r typecheck + hidden_commands: + - pnpm test:e2e:audit-log +required_artifacts: + - prediction.patch + - final_summary.md + - tests_run.md + - risks.md +metrics_profile: + - patch + - cost + - paperclip-control-plane + - liveness + - reviewability +tags: + - native + - smoke diff --git a/evals/bench/companies/README.md b/evals/bench/companies/README.md new file mode 100644 index 00000000000..13af4455a43 --- /dev/null +++ b/evals/bench/companies/README.md @@ -0,0 +1,3 @@ +# Starter Companies + +Starter company packages seed Paperclip-native tasks with a known initial org state. These directories follow the Agent Companies markdown package shape and add a provisional `.paperclip.yaml` file for benchmark-specific hints. diff --git a/evals/bench/companies/ceo-only/.paperclip.yaml b/evals/bench/companies/ceo-only/.paperclip.yaml new file mode 100644 index 00000000000..bac45c2f6bb --- /dev/null +++ b/evals/bench/companies/ceo-only/.paperclip.yaml @@ -0,0 +1,5 @@ +version: 1 +company_name: CEO Only +benchmark: + enable_agent_timers: false + intended_runner: paperclip diff --git a/evals/bench/companies/ceo-only/COMPANY.md b/evals/bench/companies/ceo-only/COMPANY.md new file mode 100644 index 00000000000..75763fea955 --- /dev/null +++ b/evals/bench/companies/ceo-only/COMPANY.md @@ -0,0 +1,7 @@ +# CEO Only + +Purpose: minimal bootstrap company for imported or direct-agent style tasks. + +Roles: + +- CEO owns the single benchmark issue. diff --git a/evals/bench/companies/ceo-only/agents/ceo/AGENTS.md b/evals/bench/companies/ceo-only/agents/ceo/AGENTS.md new file mode 100644 index 00000000000..2153dd61f46 --- /dev/null +++ b/evals/bench/companies/ceo-only/agents/ceo/AGENTS.md @@ -0,0 +1,9 @@ +# CEO + +You own the benchmark issue directly. + +Default expectations: + +- read the task prompt +- do the work or delegate when the harness allows it +- finish with reviewable artifacts diff --git a/evals/bench/companies/eng-squad-basic/.paperclip.yaml b/evals/bench/companies/eng-squad-basic/.paperclip.yaml new file mode 100644 index 00000000000..9342ba1dbdf --- /dev/null +++ b/evals/bench/companies/eng-squad-basic/.paperclip.yaml @@ -0,0 +1,6 @@ +version: 1 +company_name: Engineering Squad Basic +benchmark: + enable_agent_timers: false + intended_runner: paperclip + default_idle_stability_minutes: 5 diff --git a/evals/bench/companies/eng-squad-basic/COMPANY.md b/evals/bench/companies/eng-squad-basic/COMPANY.md new file mode 100644 index 00000000000..4967e170632 --- /dev/null +++ b/evals/bench/companies/eng-squad-basic/COMPANY.md @@ -0,0 +1,10 @@ +# Engineering Squad Basic + +Purpose: small delivery-focused company package for Paperclip-native smoke tasks. + +Roles: + +- CEO for coordination +- CTO for task routing and technical review +- Senior Engineer for implementation +- QA for verification diff --git a/evals/bench/companies/eng-squad-basic/agents/ceo/AGENTS.md b/evals/bench/companies/eng-squad-basic/agents/ceo/AGENTS.md new file mode 100644 index 00000000000..2b226bd924f --- /dev/null +++ b/evals/bench/companies/eng-squad-basic/agents/ceo/AGENTS.md @@ -0,0 +1,3 @@ +# CEO + +Coordinate the benchmark mission, keep work moving, and ensure the final artifact set is reviewable. diff --git a/evals/bench/companies/eng-squad-basic/agents/cto/AGENTS.md b/evals/bench/companies/eng-squad-basic/agents/cto/AGENTS.md new file mode 100644 index 00000000000..528f7b2f759 --- /dev/null +++ b/evals/bench/companies/eng-squad-basic/agents/cto/AGENTS.md @@ -0,0 +1,3 @@ +# CTO + +Break the mission into concrete engineering work, keep technical reviews crisp, and avoid unnecessary task churn. diff --git a/evals/bench/companies/eng-squad-basic/agents/qa/AGENTS.md b/evals/bench/companies/eng-squad-basic/agents/qa/AGENTS.md new file mode 100644 index 00000000000..324dfc135ef --- /dev/null +++ b/evals/bench/companies/eng-squad-basic/agents/qa/AGENTS.md @@ -0,0 +1,3 @@ +# QA + +Validate the deliverable against visible checks, inspect the final summary, and report residual risks. diff --git a/evals/bench/companies/eng-squad-basic/agents/senior-engineer/AGENTS.md b/evals/bench/companies/eng-squad-basic/agents/senior-engineer/AGENTS.md new file mode 100644 index 00000000000..6c34ab3d9f6 --- /dev/null +++ b/evals/bench/companies/eng-squad-basic/agents/senior-engineer/AGENTS.md @@ -0,0 +1,3 @@ +# Senior Engineer + +Implement the requested change, run relevant validation, and leave clear handoff notes for review. diff --git a/evals/bench/companies/governed-eng-squad/.paperclip.yaml b/evals/bench/companies/governed-eng-squad/.paperclip.yaml new file mode 100644 index 00000000000..fe793e97bdd --- /dev/null +++ b/evals/bench/companies/governed-eng-squad/.paperclip.yaml @@ -0,0 +1,6 @@ +version: 1 +company_name: Governed Engineering Squad +benchmark: + enable_agent_timers: false + intended_runner: paperclip + requires_review_handoffs: true diff --git a/evals/bench/companies/governed-eng-squad/COMPANY.md b/evals/bench/companies/governed-eng-squad/COMPANY.md new file mode 100644 index 00000000000..1de6b54cb9b --- /dev/null +++ b/evals/bench/companies/governed-eng-squad/COMPANY.md @@ -0,0 +1,12 @@ +# Governed Engineering Squad + +Purpose: review-heavy starter package for blocker and handoff benchmarks. + +Roles: + +- CEO +- CTO +- Senior Engineer +- Security Engineer +- UX Designer +- QA diff --git a/evals/bench/companies/governed-eng-squad/agents/ceo/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/ceo/AGENTS.md new file mode 100644 index 00000000000..d80943b1cf0 --- /dev/null +++ b/evals/bench/companies/governed-eng-squad/agents/ceo/AGENTS.md @@ -0,0 +1,3 @@ +# CEO + +Own the benchmark outcome, keep review dependencies explicit, and avoid hidden state. diff --git a/evals/bench/companies/governed-eng-squad/agents/cto/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/cto/AGENTS.md new file mode 100644 index 00000000000..ac431e5b207 --- /dev/null +++ b/evals/bench/companies/governed-eng-squad/agents/cto/AGENTS.md @@ -0,0 +1,3 @@ +# CTO + +Manage implementation sequencing, review dependencies, and blocker hygiene. diff --git a/evals/bench/companies/governed-eng-squad/agents/qa/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/qa/AGENTS.md new file mode 100644 index 00000000000..01a2e545a75 --- /dev/null +++ b/evals/bench/companies/governed-eng-squad/agents/qa/AGENTS.md @@ -0,0 +1,3 @@ +# QA + +Validate acceptance criteria, inspect artifacts, and report whether the mission is ready for review. diff --git a/evals/bench/companies/governed-eng-squad/agents/security-engineer/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/security-engineer/AGENTS.md new file mode 100644 index 00000000000..095861a673a --- /dev/null +++ b/evals/bench/companies/governed-eng-squad/agents/security-engineer/AGENTS.md @@ -0,0 +1,3 @@ +# Security Engineer + +Review security-sensitive changes, document required controls, and block work explicitly when requirements are incomplete. diff --git a/evals/bench/companies/governed-eng-squad/agents/senior-engineer/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/senior-engineer/AGENTS.md new file mode 100644 index 00000000000..0ac44ad08f1 --- /dev/null +++ b/evals/bench/companies/governed-eng-squad/agents/senior-engineer/AGENTS.md @@ -0,0 +1,3 @@ +# Senior Engineer + +Implement the technical work after required reviews are in place and leave evidence for QA. diff --git a/evals/bench/companies/governed-eng-squad/agents/ux-designer/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/ux-designer/AGENTS.md new file mode 100644 index 00000000000..a10d3eb85a6 --- /dev/null +++ b/evals/bench/companies/governed-eng-squad/agents/ux-designer/AGENTS.md @@ -0,0 +1,3 @@ +# UX Designer + +Review operator-facing states and copy so benchmark tasks can measure design-review handoffs. diff --git a/evals/bench/fixtures/README.md b/evals/bench/fixtures/README.md new file mode 100644 index 00000000000..eb3c43e6df7 --- /dev/null +++ b/evals/bench/fixtures/README.md @@ -0,0 +1,3 @@ +# Fixtures + +Fixture repos and shared env examples for benchmark tasks live here. Real fixture repositories land in later phases; Phase 1 keeps stable placeholders so tasks can reference durable paths. diff --git a/evals/bench/fixtures/repos/demo-saas/.gitkeep b/evals/bench/fixtures/repos/demo-saas/.gitkeep new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/evals/bench/fixtures/repos/demo-saas/.gitkeep @@ -0,0 +1 @@ + diff --git a/evals/bench/fixtures/repos/workflow-demo/.gitkeep b/evals/bench/fixtures/repos/workflow-demo/.gitkeep new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/evals/bench/fixtures/repos/workflow-demo/.gitkeep @@ -0,0 +1 @@ + diff --git a/evals/bench/fixtures/secrets.example.env b/evals/bench/fixtures/secrets.example.env new file mode 100644 index 00000000000..0a4b8a3c50f --- /dev/null +++ b/evals/bench/fixtures/secrets.example.env @@ -0,0 +1,5 @@ +# Example env file for local benchmark runs. +# Real values should be injected per run, not committed. +OPENAI_API_KEY= +ANTHROPIC_API_KEY= +PAPERCLIP_API_URL=http://localhost:3100 diff --git a/evals/bench/package.json b/evals/bench/package.json new file mode 100644 index 00000000000..bfa6e4a0c8a --- /dev/null +++ b/evals/bench/package.json @@ -0,0 +1,16 @@ +{ + "name": "paperclip-bench", + "private": true, + "version": "0.0.0", + "type": "module", + "description": "Scaffold for Paperclip benchmark suites, tasks, and runner assets.", + "scripts": { + "validate:manifests": "node ./scripts/validate-manifests.mjs" + }, + "devDependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "yaml": "^2.8.1" + }, + "packageManager": "pnpm@9.15.4" +} diff --git a/evals/bench/pyproject.toml b/evals/bench/pyproject.toml new file mode 100644 index 00000000000..4b1bc34906d --- /dev/null +++ b/evals/bench/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "paperclip-bench-python" +version = "0.0.0" +description = "Placeholder bridge package for paperclip-bench Python evaluator integrations." +readme = "README.md" +requires-python = ">=3.11" +dependencies = [] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" diff --git a/evals/bench/scripts/validate-manifests.mjs b/evals/bench/scripts/validate-manifests.mjs new file mode 100644 index 00000000000..4c9c99f3b5f --- /dev/null +++ b/evals/bench/scripts/validate-manifests.mjs @@ -0,0 +1,99 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import Ajv from "ajv"; +import addFormats from "ajv-formats"; +import YAML from "yaml"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.resolve(__dirname, ".."); +const schemaPath = path.join(rootDir, "bench.schema.json"); + +async function main() { + const schema = JSON.parse(await fs.readFile(schemaPath, "utf8")); + const ajv = new Ajv({ allErrors: true, strict: false }); + addFormats(ajv); + + const validateTask = ajv.compile(schema.$defs.taskV1); + const validateSuite = ajv.compile(schema.$defs.suiteV1); + + const manifestPaths = [ + ...(await collectYamlFiles(path.join(rootDir, "suites"))), + ...(await collectYamlFiles(path.join(rootDir, "benchmarks"))), + ]; + + if (manifestPaths.length === 0) { + throw new Error("No YAML manifests found to validate."); + } + + let failures = 0; + + for (const manifestPath of manifestPaths) { + const relativePath = path.relative(rootDir, manifestPath); + const doc = YAML.parse(await fs.readFile(manifestPath, "utf8")); + if (!doc || typeof doc !== "object") { + console.error(`FAIL ${relativePath}: expected YAML object`); + failures += 1; + continue; + } + + const schemaName = doc.schema; + if (schemaName === "paperclip-bench/task/v1") { + if (!validateTask(doc)) { + console.error(formatErrors(relativePath, validateTask.errors ?? [])); + failures += 1; + continue; + } + console.log(`OK ${relativePath} (task)`); + continue; + } + + if (schemaName === "paperclip-bench/suite/v1") { + if (!validateSuite(doc)) { + console.error(formatErrors(relativePath, validateSuite.errors ?? [])); + failures += 1; + continue; + } + console.log(`OK ${relativePath} (suite)`); + continue; + } + } + + if (failures > 0) { + process.exitCode = 1; + return; + } + + console.log(`Validated ${manifestPaths.length} manifest(s).`); +} + +async function collectYamlFiles(dir) { + const entries = await fs.readdir(dir, { withFileTypes: true }); + const files = []; + for (const entry of entries) { + const entryPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...(await collectYamlFiles(entryPath))); + continue; + } + if (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml")) { + files.push(entryPath); + } + } + return files; +} + +function formatErrors(relativePath, errors) { + const lines = [`FAIL ${relativePath}`]; + for (const error of errors) { + const instancePath = error.instancePath || "/"; + lines.push(` - ${instancePath}: ${error.message}`); + } + return lines.join("\n"); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/evals/bench/src/README.md b/evals/bench/src/README.md new file mode 100644 index 00000000000..4d8b99f002b --- /dev/null +++ b/evals/bench/src/README.md @@ -0,0 +1,11 @@ +# TypeScript Runtime Layout + +These directories reserve stable homes for the future benchmark CLI and runtime code. + +- `cli/`: user-facing commands +- `core/`: manifest loading and run orchestration +- `datasets/`: imported benchmark normalization +- `runners/`: Paperclip, direct-agent, noop, and other execution backends +- `evaluators/`: command and benchmark-specific evaluator adapters +- `metrics/`: metric extraction and failure taxonomy +- `reports/`: aggregate report generation diff --git a/evals/bench/src/cli/.gitkeep b/evals/bench/src/cli/.gitkeep new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/evals/bench/src/cli/.gitkeep @@ -0,0 +1 @@ + diff --git a/evals/bench/suites/paperclip-native-smoke.yaml b/evals/bench/suites/paperclip-native-smoke.yaml new file mode 100644 index 00000000000..340c4aba67d --- /dev/null +++ b/evals/bench/suites/paperclip-native-smoke.yaml @@ -0,0 +1,24 @@ +$schema: ../bench.schema.json#/$defs/suiteV1 +schema: paperclip-bench/suite/v1 +id: paperclip-native-smoke +title: Paperclip-native smoke +description: Small native task suite for control-plane behavior, artifact generation, and starter-company wiring. +runner: + kind: paperclip + mode: isolated-instance +paperclip_version: current +model: anthropic/claude-sonnet-4.5 +tasks: + - ../benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml +defaults: + budget: + max_wall_clock_minutes: 120 + max_cost_usd: 10 + max_tool_calls: 300 + max_test_runs: 20 + stop: + stable_idle_minutes: 5 + require_terminal_issue_state: true + allow_human_intervention: false +notes: + - Uses the eng-squad-basic starter company package. diff --git a/evals/bench/suites/smoke.local.yaml b/evals/bench/suites/smoke.local.yaml new file mode 100644 index 00000000000..eedb236183d --- /dev/null +++ b/evals/bench/suites/smoke.local.yaml @@ -0,0 +1,20 @@ +$schema: ../bench.schema.json#/$defs/suiteV1 +schema: paperclip-bench/suite/v1 +id: smoke.local +title: Local scaffold smoke +description: Validate the manifest contract and exercise both imported and native task lanes with a placeholder runner. +runner: + kind: noop +paperclip_version: current +model: noop +tasks: + - ../benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml + - ../benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml +defaults: + budget: + max_wall_clock_minutes: 15 + max_cost_usd: 0 + stop: + stable_idle_minutes: 1 +notes: + - Phase 1 smoke suite for schema validation and scaffold review. diff --git a/evals/bench/suites/swebench-lite-smoke.yaml b/evals/bench/suites/swebench-lite-smoke.yaml new file mode 100644 index 00000000000..f411fade448 --- /dev/null +++ b/evals/bench/suites/swebench-lite-smoke.yaml @@ -0,0 +1,22 @@ +$schema: ../bench.schema.json#/$defs/suiteV1 +schema: paperclip-bench/suite/v1 +id: swebench-lite-smoke +title: SWE-bench Lite smoke +description: Minimal imported-task suite that proves where SWE-bench Lite manifests belong and how they are referenced. +runner: + kind: prepare + mode: isolated-instance +paperclip_version: current +model: anthropic/claude-sonnet-4.5 +tasks: + - ../benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml +defaults: + budget: + max_wall_clock_minutes: 60 + max_cost_usd: 5 + max_tool_calls: 150 + stop: + stable_idle_minutes: 5 + require_terminal_issue_state: true +notes: + - Emits normalized metadata for imported tasks before the evaluator bridge lands. diff --git a/evals/bench/tasks/paperclip-native-smoke.jsonl b/evals/bench/tasks/paperclip-native-smoke.jsonl new file mode 100644 index 00000000000..76e4dbf88e5 --- /dev/null +++ b/evals/bench/tasks/paperclip-native-smoke.jsonl @@ -0,0 +1 @@ +{"id":"paperclip-native-001","title":"Add request IDs to a checkout flow","problem_statement":"In a small Paperclip-like service, add request IDs to the issue checkout path so every checkout mutation can be correlated with logs and final review notes.","time_budget_minutes":60,"token_budget_usd":5,"success_checks":["Unit tests for request ID propagation pass","Final summary includes verification and remaining risks"],"human_rubric":{"architecture":5,"minimality":5,"test_quality":5,"final_summary":5},"required_artifacts":["implementation patch","test evidence","final PR summary","risk notes"]} diff --git a/package.json b/package.json index d470705e7c3..312298281d2 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "test:e2e:headed": "npx playwright test --config tests/e2e/playwright.config.ts --headed", "test:e2e:multiuser-authenticated": "npx playwright test --config tests/e2e/playwright-multiuser-authenticated.config.ts", "evals:smoke": "cd evals/promptfoo && npx promptfoo@0.103.3 eval", + "bench:smoke": "pnpm paperclipai bench run --tasks evals/bench/tasks/paperclip-native-smoke.jsonl --benchmark paperclip_native_smoke --paperclip-version current --model noop --output .tmp/paperclip-bench-smoke --clean", "test:release-smoke": "npx playwright test --config tests/release-smoke/playwright.config.ts", "test:release-smoke:headed": "npx playwright test --config tests/release-smoke/playwright.config.ts --headed", "metrics:paperclip-commits": "tsx scripts/paperclip-commit-metrics.ts"