diff --git a/cli/src/__tests__/bench.test.ts b/cli/src/__tests__/bench.test.ts deleted file mode 100644 index 7a671b7a35b..00000000000 --- a/cli/src/__tests__/bench.test.ts +++ /dev/null @@ -1,345 +0,0 @@ -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { describe, expect, it } from "vitest"; -import { - loadBenchTasks, - normalizeBenchRunner, - normalizeBenchTask, - renderBenchIssueMarkdown, - runPaperclipBench, -} from "../commands/bench.js"; - -describe("normalizeBenchTask", () => { - it("accepts MissionBench-style snake_case fields", () => { - const task = normalizeBenchTask( - { - id: "missionbench-001", - title: "Add audit logs", - repo: "github.com/paperclipai/missionbench-saas", - starting_commit: "abc123", - problem_statement: "Add organization-level audit logs.", - time_budget_minutes: 180, - token_budget_usd: 20, - success_checks: ["npm test", "npm run typecheck"], - human_rubric: { architecture: 5, test_quality: 5 }, - required_artifacts: ["implementation patch", "risk notes"], - }, - { lineLabel: "tasks.jsonl:1" }, - ); - - expect(task).toMatchObject({ - id: "missionbench-001", - title: "Add audit logs", - problemStatement: "Add organization-level audit logs.", - repo: { - url: "github.com/paperclipai/missionbench-saas", - baseCommit: "abc123", - }, - maxWallClockMinutes: 180, - maxCostUsd: 20, - successChecks: ["npm test", "npm run typecheck"], - requiredArtifacts: ["implementation patch", "risk notes"], - }); - expect(task.humanRubric).toEqual([ - { name: "architecture", maxScore: 5 }, - { name: "test_quality", maxScore: 5 }, - ]); - }); - - it("rejects tasks without a problem statement", () => { - expect(() => - normalizeBenchTask( - { - id: "missing-problem", - title: "Missing problem", - }, - { lineLabel: "tasks.jsonl:2" }, - ), - ).toThrow(/requires problemStatement/i); - }); - - it("keeps an explicit issue reference for reuse", () => { - const task = normalizeBenchTask( - { - id: "missionbench-002", - title: "Reuse benchmark issue", - problem_statement: "Continue work in an existing benchmark issue.", - paperclip_issue_id: "PAP-42", - }, - { lineLabel: "tasks.jsonl:3" }, - ); - - expect(task.paperclipIssueRef).toBe("PAP-42"); - }); -}); - -describe("normalizeBenchRunner", () => { - it("accepts prepare and paperclip", () => { - expect(normalizeBenchRunner(undefined)).toBe("prepare"); - expect(normalizeBenchRunner("prepare")).toBe("prepare"); - expect(normalizeBenchRunner("paperclip")).toBe("paperclip"); - }); - - it("rejects unsupported runners", () => { - expect(() => normalizeBenchRunner("live")).toThrow(/Unsupported bench runner/i); - }); -}); - -describe("renderBenchIssueMarkdown", () => { - it("renders a Paperclip-ready task prompt", () => { - const markdown = renderBenchIssueMarkdown( - { - id: "missionbench-001", - title: "Add audit logs", - problemStatement: "Add organization-level audit logs.", - repo: { url: "https://example.com/repo.git", baseCommit: "abc123" }, - maxWallClockMinutes: 120, - maxCostUsd: 10, - successChecks: ["npm test"], - humanRubric: [{ name: "architecture", maxScore: 5 }], - requiredArtifacts: ["final PR summary"], - }, - { - benchmark: "missionbench", - runId: "run-1", - model: "anthropic/claude-sonnet-4.5", - paperclipVersion: "current", - }, - ); - - expect(markdown).toContain("# Add audit logs"); - expect(markdown).toContain("- Task ID: `missionbench-001`"); - expect(markdown).toContain("- URL: https://example.com/repo.git"); - expect(markdown).toContain("- npm test"); - expect(markdown).toContain("- architecture (5 pts)"); - }); -}); - -describe("runPaperclipBench", () => { - it("writes manifest, per-task prompts, traces, and results", async () => { - const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-test-")); - try { - const tasksPath = path.join(tmpDir, "tasks.jsonl"); - const outputDir = path.join(tmpDir, "out"); - await writeFile( - tasksPath, - `${JSON.stringify({ - id: "task-1", - title: "Fix checkout", - problemStatement: "Make checkout idempotent.", - successChecks: ["pnpm test"], - })}\n`, - "utf8", - ); - - const manifest = await runPaperclipBench({ - tasksPath, - benchmark: "paperclip_native", - outputDir, - runId: "test-run", - paperclipVersion: "current", - model: "noop", - runner: "prepare", - clean: true, - }); - - expect(manifest.taskCount).toBe(1); - expect(manifest.tasks[0]?.issuePath).toBe("tasks/task-1/issue.md"); - - const issueMarkdown = await readFile(path.join(outputDir, "tasks/task-1/issue.md"), "utf8"); - expect(issueMarkdown).toContain("Make checkout idempotent."); - - const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8"); - expect(results).toContain('"status":"prepared"'); - - const reloadedTasks = await loadBenchTasks(tasksPath); - expect(reloadedTasks.map((task) => task.id)).toEqual(["task-1"]); - } finally { - await rm(tmpDir, { recursive: true, force: true }); - } - }); - - it("records live Paperclip runner outputs", async () => { - const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-live-test-")); - try { - const tasksPath = path.join(tmpDir, "tasks.jsonl"); - const outputDir = path.join(tmpDir, "out"); - await writeFile( - tasksPath, - `${JSON.stringify({ - id: "task-live-1", - title: "Ship audit logs", - problemStatement: "Add audit log support.", - successChecks: ["pnpm test"], - })}\n`, - "utf8", - ); - - const manifest = await runPaperclipBench({ - tasksPath, - benchmark: "paperclip_native", - outputDir, - runId: "live-run", - paperclipVersion: "current", - model: "noop", - runner: "paperclip", - clean: true, - issueProvisioner: async () => ({ - source: "created", - issue: { - id: "issue-1", - companyId: "company-1", - projectId: null, - projectWorkspaceId: null, - goalId: null, - parentId: null, - title: "Ship audit logs", - description: null, - status: "todo", - priority: "medium", - assigneeAgentId: "agent-1", - assigneeUserId: null, - checkoutRunId: null, - executionRunId: null, - executionAgentNameKey: null, - executionLockedAt: null, - createdByAgentId: null, - createdByUserId: null, - issueNumber: 1, - identifier: "PAP-1", - requestDepth: 0, - billingCode: null, - assigneeAdapterOverrides: null, - executionWorkspaceId: null, - executionWorkspacePreference: null, - executionWorkspaceSettings: null, - startedAt: null, - completedAt: null, - cancelledAt: null, - hiddenAt: null, - createdAt: new Date("2026-04-24T12:00:00Z"), - updatedAt: new Date("2026-04-24T12:00:00Z"), - }, - }), - runLiveTask: async ({ taskDir, issue }) => { - await writeFile(path.join(taskDir, "patch.diff"), "diff --git a/file b/file\n", "utf8"); - return { - issue: { - ...issue, - status: "done", - executionRunId: "run-2", - updatedAt: new Date("2026-04-24T12:02:00Z"), - }, - runIds: ["run-1", "run-2"], - status: "done", - patchPath: "tasks/task-live-1/patch.diff", - }; - }, - }); - - expect(manifest.runner).toBe("paperclip"); - expect(manifest.tasks[0]).toMatchObject({ - status: "done", - patchPath: "tasks/task-live-1/patch.diff", - paperclipRunIds: ["run-1", "run-2"], - paperclipIssue: { - id: "issue-1", - identifier: "PAP-1", - status: "done", - source: "created", - }, - }); - - const trace = await readFile(path.join(outputDir, "tasks/task-live-1/trace.json"), "utf8"); - expect(trace).toContain('"paperclipRunIds": ['); - expect(trace).toContain('"patchPath": "tasks/task-live-1/patch.diff"'); - - const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8"); - expect(results).toContain('"status":"done"'); - expect(results).toContain('"paperclipRunIds":["run-1","run-2"]'); - expect(results).toContain('"patchPath":"tasks/task-live-1/patch.diff"'); - } finally { - await rm(tmpDir, { recursive: true, force: true }); - } - }); - - it("writes failed live-run results when the Paperclip runner errors", async () => { - const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-live-fail-test-")); - try { - const tasksPath = path.join(tmpDir, "tasks.jsonl"); - const outputDir = path.join(tmpDir, "out"); - await writeFile( - tasksPath, - `${JSON.stringify({ - id: "task-live-2", - title: "Fail audit logs", - problemStatement: "Add audit log support.", - successChecks: ["pnpm test"], - })}\n`, - "utf8", - ); - - const manifest = await runPaperclipBench({ - tasksPath, - benchmark: "paperclip_native", - outputDir, - runId: "live-fail-run", - paperclipVersion: "current", - model: "noop", - runner: "paperclip", - clean: true, - continueOnError: true, - issueProvisioner: async () => ({ - source: "reused", - issue: { - id: "issue-2", - companyId: "company-1", - projectId: null, - projectWorkspaceId: null, - goalId: null, - parentId: null, - title: "Fail audit logs", - description: null, - status: "in_progress", - priority: "medium", - assigneeAgentId: "agent-1", - assigneeUserId: null, - checkoutRunId: null, - executionRunId: null, - executionAgentNameKey: null, - executionLockedAt: null, - createdByAgentId: null, - createdByUserId: null, - issueNumber: 2, - identifier: "PAP-2", - requestDepth: 0, - billingCode: null, - assigneeAdapterOverrides: null, - executionWorkspaceId: null, - executionWorkspacePreference: null, - executionWorkspaceSettings: null, - startedAt: null, - completedAt: null, - cancelledAt: null, - hiddenAt: null, - createdAt: new Date("2026-04-24T12:00:00Z"), - updatedAt: new Date("2026-04-24T12:00:00Z"), - }, - }), - runLiveTask: async () => { - throw new Error("heartbeat invoke failed"); - }, - }); - - expect(manifest.tasks[0]?.status).toBe("failed"); - - const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8"); - expect(results).toContain('"status":"failed"'); - expect(results).toContain('"failureStage":"paperclip_runner"'); - expect(results).toContain('"notes":"heartbeat invoke failed"'); - } finally { - await rm(tmpDir, { recursive: true, force: true }); - } - }); -}); diff --git a/cli/src/commands/bench.ts b/cli/src/commands/bench.ts deleted file mode 100644 index 2840798fc7c..00000000000 --- a/cli/src/commands/bench.ts +++ /dev/null @@ -1,1277 +0,0 @@ -import { spawnSync } from "node:child_process"; -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; -import path from "node:path"; -import { setTimeout as delay } from "node:timers/promises"; -import pc from "picocolors"; -import { Command } from "commander"; -import type { AgentWakeupResponse, HeartbeatRun, Issue } from "@paperclipai/shared"; -import { - addCommonClientOptions, - handleCommandError, - printOutput, - resolveCommandContext, - type BaseClientOptions, -} from "./client/common.js"; -import { ApiRequestError } from "../client/http.js"; - -export interface BenchRepoSpec { - url?: string; - path?: string; - baseCommit?: string; -} - -export interface BenchRubricItem { - name: string; - description?: string; - maxScore?: number; -} - -export interface BenchTask { - id: string; - title: string; - problemStatement: string; - repo?: BenchRepoSpec; - paperclipIssueRef?: string; - maxWallClockMinutes?: number; - maxCostUsd?: number; - successChecks: string[]; - humanRubric: BenchRubricItem[]; - requiredArtifacts: string[]; - metadata?: Record; -} - -export interface BenchManifestTask { - id: string; - title: string; - issuePath: string; - taskPath: string; - tracePath: string; - patchPath?: string; - workspacePath?: string; - repo?: BenchRepoSpec; - successChecks: string[]; - requiredArtifacts: string[]; - paperclipRunIds?: string[]; - status?: BenchTaskStatus; - paperclipIssue?: { - id: string; - identifier?: string; - status?: string; - source?: "created" | "reused"; - }; -} - -export type BenchRunner = "prepare" | "paperclip"; -export type BenchTaskStatus = "prepared" | "issue_created" | "issue_reused" | Issue["status"] | "failed"; - -export interface BenchManifest { - schemaVersion: 1; - runId: string; - benchmark: string; - paperclipVersion: string; - model: string; - runner: BenchRunner; - createdAt: string; - outputDir: string; - maxCostUsd?: number; - maxWallClockMinutes?: number; - taskCount: number; - tasks: BenchManifestTask[]; -} - -export interface BenchResult { - runId: string; - benchmark: string; - taskId: string; - status: BenchTaskStatus; - issuePath: string; - taskPath: string; - tracePath: string; - patchPath?: string; - workspacePath?: string; - paperclipIssueId?: string; - paperclipIssueIdentifier?: string; - paperclipRunIds?: string[]; - failureStage?: string | null; - notes?: string; -} - -interface BenchRunOptions extends BaseClientOptions { - tasks: string; - benchmark?: string; - output?: string; - runId?: string; - paperclipVersion?: string; - model?: string; - runner?: string; - maxCostUsd?: string; - maxWallClockMinutes?: string; - maxTasks?: string; - clean?: boolean; - checkout?: boolean; - createIssues?: boolean; - projectId?: string; - goalId?: string; - assigneeAgentId?: string; - issueStatus?: string; - billingCode?: string; - continueOnError?: boolean; -} - -interface BenchValidateOptions extends BaseClientOptions { - tasks: string; - maxTasks?: string; -} - -interface NormalizationDefaults { - maxCostUsd?: number; - maxWallClockMinutes?: number; -} - -interface RunPaperclipBenchOptions { - tasksPath: string; - benchmark: string; - outputDir: string; - runId: string; - paperclipVersion: string; - model: string; - runner: BenchRunner; - maxCostUsd?: number; - maxWallClockMinutes?: number; - maxTasks?: number; - clean?: boolean; - checkout?: boolean; - issueProvisioner?: (task: BenchTask, issueMarkdown: string) => Promise; - runLiveTask?: (input: BenchLiveTaskInput) => Promise; - continueOnError?: boolean; -} - -interface BenchIssueResolution { - issue: Issue; - source: "created" | "reused"; -} - -interface BenchLiveTaskInput { - task: BenchTask; - taskDir: string; - outputDir: string; - issueMarkdown: string; - issue: Issue; - issueSource: "created" | "reused"; - benchmark: string; - runId: string; - startedAt: string; - defaultMaxCostUsd?: number; - defaultMaxWallClockMinutes?: number; -} - -interface BenchLiveTaskOutcome { - issue: Issue; - runIds: string[]; - status: BenchTaskStatus; - patchPath?: string; - failureStage?: string | null; - notes?: string; -} - -class BenchStageError extends Error { - readonly stage: string; - - constructor(stage: string, message: string) { - super(message); - this.stage = stage; - } -} - -const TERMINAL_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]); -const TERMINAL_ISSUE_STATUSES = new Set(["done", "blocked", "cancelled", "in_review"]); -const BENCH_POLL_INTERVAL_MS = 1000; - -export function registerBenchCommands(program: Command): void { - const bench = program - .command("bench") - .description("Prepare and run Paperclip benchmark task suites"); - - addCommonClientOptions( - bench - .command("run") - .description("Normalize a JSONL benchmark suite into Paperclip-ready run artifacts") - .requiredOption("--tasks ", "JSONL task file") - .option("--benchmark ", "Benchmark name", "paperclip_native") - .option("--output ", "Run output directory") - .option("--run-id ", "Stable run id for repeatable output paths") - .option("--paperclip-version ", "Paperclip version label", "current") - .option("--model ", "Model or bundle label", "unspecified") - .option("--runner ", "Runner mode: prepare or paperclip", "prepare") - .option("--max-cost-usd ", "Default per-task cost budget") - .option("--max-wall-clock-minutes ", "Default per-task wall-clock budget") - .option("--max-tasks ", "Limit the number of tasks read from the suite") - .option("--clean", "Remove the output directory before writing") - .option("--checkout", "Clone/check out task repositories into each task workspace") - .option("--create-issues", "Create one Paperclip issue per benchmark task") - .option("--project-id ", "Project ID for created benchmark issues") - .option("--goal-id ", "Goal ID for created benchmark issues") - .option("--assignee-agent-id ", "Assignee agent ID for created benchmark issues") - .option("--issue-status ", "Status for created benchmark issues", "todo") - .option("--billing-code ", "Billing code for created benchmark issues") - .option("--continue-on-error", "Write failed task results and continue") - .action(async (opts: BenchRunOptions) => { - try { - const runner = normalizeBenchRunner(opts.runner); - const benchmark = normalizeIdentifier(opts.benchmark ?? "paperclip_native", "benchmark"); - const runId = opts.runId?.trim() || buildDefaultRunId(benchmark); - const outputDir = opts.output?.trim() || path.join(".tmp", "paperclip-bench", runId); - const maxCostUsd = parseOptionalPositiveNumber(opts.maxCostUsd, "max-cost-usd"); - const maxWallClockMinutes = parseOptionalPositiveNumber( - opts.maxWallClockMinutes, - "max-wall-clock-minutes", - ); - const maxTasks = parseOptionalPositiveInteger(opts.maxTasks, "max-tasks"); - - const issueProvisioner = opts.createIssues || runner === "paperclip" - ? buildIssueProvisioner({ - ...opts, - benchmark, - runId, - }) - : undefined; - const runLiveTask = runner === "paperclip" - ? buildPaperclipLiveTaskRunner({ - ...opts, - benchmark, - runId, - }) - : undefined; - - const manifest = await runPaperclipBench({ - tasksPath: opts.tasks, - benchmark, - outputDir, - runId, - paperclipVersion: opts.paperclipVersion?.trim() || "current", - model: opts.model?.trim() || "unspecified", - runner, - maxCostUsd, - maxWallClockMinutes, - maxTasks, - clean: opts.clean, - checkout: opts.checkout, - issueProvisioner, - runLiveTask, - continueOnError: opts.continueOnError, - }); - - if (opts.json) { - printOutput(manifest, { json: true }); - return; - } - - console.log(pc.bold(runner === "prepare" ? "Paperclip bench run prepared" : "Paperclip bench run completed")); - console.log(`Run: ${manifest.runId}`); - console.log(`Benchmark: ${manifest.benchmark}`); - console.log(`Runner: ${manifest.runner}`); - console.log(`Tasks: ${manifest.taskCount}`); - console.log(`Output: ${manifest.outputDir}`); - if (issueProvisioner) { - const created = manifest.tasks.filter((task) => task.paperclipIssue?.source === "created").length; - const reused = manifest.tasks.filter((task) => task.paperclipIssue?.source === "reused").length; - if (created > 0) console.log(`Issues created: ${created}`); - if (reused > 0) console.log(`Issues reused: ${reused}`); - } - } catch (err) { - handleCommandError(err); - } - }), - { includeCompany: true }, - ); - - addCommonClientOptions( - bench - .command("validate") - .description("Validate a JSONL benchmark task suite") - .requiredOption("--tasks ", "JSONL task file") - .option("--max-tasks ", "Limit the number of tasks read from the suite") - .action(async (opts: BenchValidateOptions) => { - try { - const maxTasks = parseOptionalPositiveInteger(opts.maxTasks, "max-tasks"); - const tasks = await loadBenchTasks(opts.tasks, {}, maxTasks); - - if (opts.json) { - printOutput({ taskCount: tasks.length, taskIds: tasks.map((task) => task.id) }, { json: true }); - return; - } - - console.log(pc.bold("Benchmark suite is valid")); - console.log(`Tasks: ${tasks.length}`); - for (const task of tasks) { - console.log(`- ${task.id}: ${task.title}`); - } - } catch (err) { - handleCommandError(err); - } - }), - ); -} - -export async function runPaperclipBench(options: RunPaperclipBenchOptions): Promise { - const defaults: NormalizationDefaults = { - maxCostUsd: options.maxCostUsd, - maxWallClockMinutes: options.maxWallClockMinutes, - }; - const tasks = await loadBenchTasks(options.tasksPath, defaults, options.maxTasks); - const outputDir = path.resolve(options.outputDir); - - if (options.clean) { - await rm(outputDir, { recursive: true, force: true }); - } - await mkdir(outputDir, { recursive: true }); - - const createdAt = new Date().toISOString(); - const results: BenchResult[] = []; - const manifestTasks: BenchManifestTask[] = []; - - for (const task of tasks) { - const safeId = safePathSegment(task.id); - const taskDir = path.join(outputDir, "tasks", safeId); - await mkdir(taskDir, { recursive: true }); - - const issuePath = path.join(taskDir, "issue.md"); - const taskPath = path.join(taskDir, "task.json"); - const tracePath = path.join(taskDir, "trace.json"); - const workspacePath = options.checkout ? path.join(taskDir, "repo") : undefined; - const issueMarkdown = renderBenchIssueMarkdown(task, { - benchmark: options.benchmark, - runId: options.runId, - model: options.model, - paperclipVersion: options.paperclipVersion, - }); - - const startedAt = new Date().toISOString(); - const events: Array> = [ - { at: startedAt, type: "task.prepared", taskId: task.id }, - ]; - - let paperclipIssue: BenchManifestTask["paperclipIssue"]; - let paperclipRunIds: string[] | undefined; - let patchPath: string | undefined; - let status: BenchResult["status"] = "prepared"; - let failureStage: string | null = null; - let notes: string | undefined; - - try { - await writeFile(issuePath, issueMarkdown, "utf8"); - await writeFile(taskPath, JSON.stringify(task, null, 2), "utf8"); - - if (options.checkout) { - if (!task.repo?.url && !task.repo?.path) { - throw new Error(`Task ${task.id} cannot be checked out because it has no repo.url or repo.path`); - } - checkoutTaskRepo(task, workspacePath!); - events.push({ at: new Date().toISOString(), type: "repo.checked_out", path: workspacePath }); - } - - let resolvedIssue: Issue | undefined; - if (options.issueProvisioner) { - const issueResolution = await options.issueProvisioner(task, issueMarkdown); - resolvedIssue = issueResolution.issue; - paperclipIssue = { - id: issueResolution.issue.id, - identifier: issueResolution.issue.identifier ?? undefined, - status: issueResolution.issue.status ?? undefined, - source: issueResolution.source, - }; - status = issueResolution.source === "created" ? "issue_created" : "issue_reused"; - events.push({ - at: new Date().toISOString(), - type: issueResolution.source === "created" ? "paperclip.issue_created" : "paperclip.issue_reused", - issueId: issueResolution.issue.id, - identifier: issueResolution.issue.identifier, - }); - } - - if (options.runner === "paperclip") { - if (!resolvedIssue) { - throw new BenchStageError( - "issue_provision", - `Runner "${options.runner}" requires an issue provisioner or a task row with an existing issue reference`, - ); - } - if (!options.runLiveTask) { - throw new BenchStageError("paperclip_runner", "Live Paperclip runner is not configured"); - } - - const liveOutcome = await options.runLiveTask({ - task, - taskDir, - outputDir, - issueMarkdown, - issue: resolvedIssue, - issueSource: paperclipIssue?.source ?? "created", - benchmark: options.benchmark, - runId: options.runId, - startedAt, - defaultMaxCostUsd: options.maxCostUsd, - defaultMaxWallClockMinutes: options.maxWallClockMinutes, - }); - paperclipIssue = { - id: liveOutcome.issue.id, - identifier: liveOutcome.issue.identifier ?? undefined, - status: liveOutcome.issue.status ?? undefined, - source: paperclipIssue?.source, - }; - paperclipRunIds = liveOutcome.runIds; - patchPath = liveOutcome.patchPath; - status = liveOutcome.status; - failureStage = liveOutcome.failureStage ?? null; - notes = liveOutcome.notes; - events.push({ - at: new Date().toISOString(), - type: "paperclip.task_completed", - issueId: liveOutcome.issue.id, - issueStatus: liveOutcome.issue.status, - runIds: liveOutcome.runIds, - patchPath: liveOutcome.patchPath ?? null, - failureStage: liveOutcome.failureStage ?? null, - }); - } - } catch (err) { - status = "failed"; - failureStage = - err instanceof BenchStageError - ? err.stage - : options.runner === "paperclip" - ? "paperclip_runner" - : options.issueProvisioner - ? "prepare_or_create_issue" - : "prepare"; - notes = err instanceof Error ? err.message : String(err); - events.push({ - at: new Date().toISOString(), - type: "task.failed", - stage: failureStage, - error: notes, - }); - - if (!options.continueOnError) { - await writeTrace(tracePath, task.id, startedAt, status, events, notes, { - paperclipIssueId: paperclipIssue?.id ?? null, - paperclipIssueIdentifier: paperclipIssue?.identifier ?? null, - paperclipRunIds: paperclipRunIds ?? [], - patchPath: patchPath ?? null, - failureStage, - }); - throw err; - } - } - - const endedAt = new Date().toISOString(); - events.push({ at: endedAt, type: "task.finished", status }); - await writeTrace(tracePath, task.id, startedAt, status, events, notes, { - paperclipIssueId: paperclipIssue?.id ?? null, - paperclipIssueIdentifier: paperclipIssue?.identifier ?? null, - paperclipRunIds: paperclipRunIds ?? [], - patchPath: patchPath ?? null, - failureStage, - }); - - manifestTasks.push({ - id: task.id, - title: task.title, - issuePath: path.relative(outputDir, issuePath), - taskPath: path.relative(outputDir, taskPath), - tracePath: path.relative(outputDir, tracePath), - patchPath, - workspacePath: workspacePath ? path.relative(outputDir, workspacePath) : undefined, - repo: task.repo, - successChecks: task.successChecks, - requiredArtifacts: task.requiredArtifacts, - paperclipRunIds, - status, - paperclipIssue, - }); - - results.push({ - runId: options.runId, - benchmark: options.benchmark, - taskId: task.id, - status, - issuePath: path.relative(outputDir, issuePath), - taskPath: path.relative(outputDir, taskPath), - tracePath: path.relative(outputDir, tracePath), - patchPath, - workspacePath: workspacePath ? path.relative(outputDir, workspacePath) : undefined, - paperclipIssueId: paperclipIssue?.id, - paperclipIssueIdentifier: paperclipIssue?.identifier, - paperclipRunIds, - failureStage, - notes, - }); - } - - const manifest: BenchManifest = { - schemaVersion: 1, - runId: options.runId, - benchmark: options.benchmark, - paperclipVersion: options.paperclipVersion, - model: options.model, - runner: options.runner, - createdAt, - outputDir, - maxCostUsd: options.maxCostUsd, - maxWallClockMinutes: options.maxWallClockMinutes, - taskCount: manifestTasks.length, - tasks: manifestTasks, - }; - - await writeFile(path.join(outputDir, "manifest.json"), JSON.stringify(manifest, null, 2), "utf8"); - await writeFile( - path.join(outputDir, "results.jsonl"), - `${results.map((result) => JSON.stringify(result)).join("\n")}\n`, - "utf8", - ); - - return manifest; -} - -export async function loadBenchTasks( - filePath: string, - defaults: NormalizationDefaults = {}, - maxTasks?: number, -): Promise { - const contents = await readFile(filePath, "utf8"); - const rawTasks = parseJsonlObjects(contents, filePath); - const selected = maxTasks === undefined ? rawTasks : rawTasks.slice(0, maxTasks); - const seen = new Set(); - - return selected.map((raw, index) => { - const task = normalizeBenchTask(raw, { - lineLabel: `${filePath}:${index + 1}`, - defaults, - }); - if (seen.has(task.id)) { - throw new Error(`Duplicate benchmark task id '${task.id}' in ${filePath}`); - } - seen.add(task.id); - return task; - }); -} - -export function normalizeBenchTask( - raw: Record, - opts: { lineLabel: string; defaults?: NormalizationDefaults }, -): BenchTask { - const defaults = opts.defaults ?? {}; - const id = firstString(raw, ["id", "task_id", "taskId"]); - const title = firstString(raw, ["title", "name"]); - const problemStatement = firstString(raw, [ - "problemStatement", - "problem_statement", - "description", - "issue", - "prompt", - "goal", - ]); - - if (!id) throw new Error(`${opts.lineLabel}: benchmark task requires id`); - normalizeIdentifier(id, "task id"); - if (!title) throw new Error(`${opts.lineLabel}: benchmark task '${id}' requires title`); - if (!problemStatement) { - throw new Error(`${opts.lineLabel}: benchmark task '${id}' requires problemStatement, description, issue, prompt, or goal`); - } - - return { - id, - title, - problemStatement, - repo: normalizeRepoSpec(raw), - paperclipIssueRef: normalizeIssueRef(raw), - maxWallClockMinutes: firstNumber(raw, [ - "maxWallClockMinutes", - "max_wall_clock_minutes", - "timeBudgetMinutes", - "time_budget_minutes", - ]) ?? defaults.maxWallClockMinutes, - maxCostUsd: firstNumber(raw, ["maxCostUsd", "max_cost_usd", "tokenBudgetUsd", "token_budget_usd"]) ?? defaults.maxCostUsd, - successChecks: firstStringArray(raw, ["successChecks", "success_checks", "checks"]), - humanRubric: normalizeRubric(raw.humanRubric ?? raw.human_rubric), - requiredArtifacts: firstStringArray(raw, ["requiredArtifacts", "required_artifacts"]), - metadata: asRecord(raw.metadata), - }; -} - -export function renderBenchIssueMarkdown( - task: BenchTask, - run: { - benchmark: string; - runId: string; - model: string; - paperclipVersion: string; - }, -): string { - const lines: string[] = [ - `# ${task.title}`, - "", - "## Benchmark Context", - "", - `- Task ID: \`${task.id}\``, - `- Benchmark: \`${run.benchmark}\``, - `- Run ID: \`${run.runId}\``, - `- Paperclip version: \`${run.paperclipVersion}\``, - `- Model or bundle: \`${run.model}\``, - ]; - - if (task.maxWallClockMinutes !== undefined || task.maxCostUsd !== undefined) { - lines.push("", "## Budgets", ""); - if (task.maxWallClockMinutes !== undefined) { - lines.push(`- Max wall-clock minutes: ${task.maxWallClockMinutes}`); - } - if (task.maxCostUsd !== undefined) { - lines.push(`- Max cost USD: ${task.maxCostUsd}`); - } - } - - if (task.repo) { - lines.push("", "## Repository", ""); - if (task.repo.url) lines.push(`- URL: ${task.repo.url}`); - if (task.repo.path) lines.push(`- Path: ${task.repo.path}`); - if (task.repo.baseCommit) lines.push(`- Base commit: ${task.repo.baseCommit}`); - } - - lines.push( - "", - "## Task", - "", - task.problemStatement.trim(), - "", - "## Rules", - "", - "- Make the minimal correct code change.", - "- Add or update tests when appropriate.", - "- Run the smallest relevant verification that proves the change.", - "- Do not modify unrelated files.", - "- At the end, produce a final summary with changes, verification, and remaining risks.", - ); - - if (task.successChecks.length > 0) { - lines.push("", "## Success Checks", ""); - lines.push(...task.successChecks.map((check) => `- ${check}`)); - } - - if (task.requiredArtifacts.length > 0) { - lines.push("", "## Required Artifacts", ""); - lines.push(...task.requiredArtifacts.map((artifact) => `- ${artifact}`)); - } - - if (task.humanRubric.length > 0) { - lines.push("", "## Human Rubric", ""); - for (const item of task.humanRubric) { - const suffix = item.maxScore === undefined ? "" : ` (${item.maxScore} pts)`; - const description = item.description ? `: ${item.description}` : ""; - lines.push(`- ${item.name}${suffix}${description}`); - } - } - - return `${lines.join("\n")}\n`; -} - -export function normalizeBenchRunner(value: string | undefined): BenchRunner { - const normalized = value?.trim() || "prepare"; - if (normalized === "prepare" || normalized === "paperclip") { - return normalized; - } - throw new Error(`Unsupported bench runner '${normalized}'. Expected one of: prepare, paperclip.`); -} - -function buildIssueProvisioner(opts: BenchRunOptions & { benchmark: string; runId: string }) { - const ctx = resolveCommandContext(opts); - return async (task: BenchTask, issueMarkdown: string): Promise => { - const issueRef = task.paperclipIssueRef?.trim(); - if (issueRef) { - const existing = await ctx.api.get(`/api/issues/${encodeURIComponent(issueRef)}`); - if (!existing) { - throw new BenchStageError("issue_provision", `Benchmark task ${task.id} references missing issue ${issueRef}`); - } - return { issue: existing, source: "reused" }; - } - - if (!ctx.companyId) { - throw new BenchStageError( - "issue_provision", - "Company ID is required to create benchmark issues. Pass --company-id, set PAPERCLIP_COMPANY_ID, or reuse an existing issue from the task row.", - ); - } - - const payload = { - title: `[${opts.benchmark}] ${task.title}`, - description: issueMarkdown, - status: opts.issueStatus ?? "todo", - priority: "medium", - assigneeAgentId: opts.assigneeAgentId, - projectId: opts.projectId, - goalId: opts.goalId, - billingCode: opts.billingCode || `bench:${opts.runId}`, - }; - const issue = await ctx.api.post(`/api/companies/${ctx.companyId}/issues`, payload); - if (!issue) { - throw new BenchStageError("issue_provision", `Paperclip API did not return the created issue for task ${task.id}`); - } - return { issue, source: "created" }; - }; -} - -function buildPaperclipLiveTaskRunner(opts: BenchRunOptions & { benchmark: string; runId: string }) { - const ctx = resolveCommandContext(opts); - - return async (input: BenchLiveTaskInput): Promise => { - const assigneeAgentId = input.issue.assigneeAgentId ?? (opts.assigneeAgentId?.trim() || null); - if (!assigneeAgentId) { - throw new BenchStageError( - "issue_checkout", - `Benchmark issue ${input.issue.identifier ?? input.issue.id} has no assigneeAgentId to run`, - ); - } - - try { - await ctx.api.post(`/api/issues/${input.issue.id}/checkout`, { - agentId: assigneeAgentId, - expectedStatuses: ["backlog", "todo", "blocked", "in_review", "in_progress"], - }); - } catch (err) { - if (err instanceof ApiRequestError) { - throw new BenchStageError("issue_checkout", err.message); - } - throw err; - } - - const wakeResponse = await ctx.api.post(`/api/agents/${assigneeAgentId}/wakeup`, { - source: "assignment", - triggerDetail: "manual", - reason: "bench_runner", - payload: { - benchmark: input.benchmark, - runId: input.runId, - taskId: input.task.id, - issueId: input.issue.id, - }, - idempotencyKey: `bench:${input.runId}:${input.task.id}:${input.issue.id}`, - }); - - const observedRunIds = new Set(); - const initialRunId = readWakeupRunId(wakeResponse); - if (initialRunId) observedRunIds.add(initialRunId); - - const budgetMs = resolveWallClockBudgetMs(input.task, input.defaultMaxWallClockMinutes); - const deadline = budgetMs ? Date.now() + budgetMs : null; - let latestIssue = await ctx.api.get(`/api/issues/${input.issue.id}`); - if (!latestIssue) { - throw new BenchStageError("issue_poll", `Benchmark issue ${input.issue.id} disappeared while polling`); - } - - let finalRun: HeartbeatRun | null = null; - let failureStage: string | null = null; - let notes = readWakeupNotes(wakeResponse); - - while (true) { - const activeRunSummary = await ctx.api.get<{ id: string } | null>(`/api/issues/${input.issue.id}/active-run`); - const activeRunId = typeof activeRunSummary?.id === "string" ? activeRunSummary.id : null; - if (activeRunId) { - observedRunIds.add(activeRunId); - const run = await ctx.api.get(`/api/heartbeat-runs/${activeRunId}`); - if (run) { - finalRun = run; - const costUsd = readHeartbeatRunCostUsd(run); - const maxCostUsd = input.task.maxCostUsd ?? input.defaultMaxCostUsd; - if ( - maxCostUsd !== undefined - && costUsd !== null - && costUsd > maxCostUsd - ) { - try { - await ctx.api.post(`/api/heartbeat-runs/${run.id}/cancel`); - } catch (cancelErr) { - notes = appendNote( - notes, - `Cost budget exceeded at ${costUsd.toFixed(2)} USD, and cancelling run ${run.id} failed: ${cancelErr instanceof Error ? cancelErr.message : String(cancelErr)}`, - ); - } - failureStage = "cost_budget_exceeded"; - notes = appendNote(notes, `Cancelled run ${run.id} after exceeding max cost budget ${maxCostUsd} USD.`); - latestIssue = (await ctx.api.get(`/api/issues/${input.issue.id}`)) ?? latestIssue; - break; - } - } - } - - latestIssue = (await ctx.api.get(`/api/issues/${input.issue.id}`)) ?? latestIssue; - if (latestIssue.executionRunId) observedRunIds.add(latestIssue.executionRunId); - - if (TERMINAL_ISSUE_STATUSES.has(latestIssue.status)) { - if (latestIssue.executionRunId) { - finalRun = await ctx.api.get(`/api/heartbeat-runs/${latestIssue.executionRunId}`, { ignoreNotFound: true }); - } - break; - } - - if (!activeRunId && latestIssue.executionRunId) { - finalRun = await ctx.api.get(`/api/heartbeat-runs/${latestIssue.executionRunId}`, { ignoreNotFound: true }); - if (finalRun?.status && TERMINAL_RUN_STATUSES.has(finalRun.status)) { - break; - } - } - - if (deadline && Date.now() >= deadline) { - failureStage = failureStage ?? "run_timeout"; - if (activeRunId) { - try { - await ctx.api.post(`/api/heartbeat-runs/${activeRunId}/cancel`); - notes = appendNote(notes, `Cancelled run ${activeRunId} after wall-clock budget expired.`); - } catch (cancelErr) { - notes = appendNote( - notes, - `Wall-clock budget expired for run ${activeRunId}, and cancelling it failed: ${cancelErr instanceof Error ? cancelErr.message : String(cancelErr)}`, - ); - } - } - latestIssue = (await ctx.api.get(`/api/issues/${input.issue.id}`)) ?? latestIssue; - break; - } - - await delay(BENCH_POLL_INTERVAL_MS); - } - - const issueRuns = await listIssueRunIdsSince(ctx, latestIssue.companyId, assigneeAgentId, latestIssue.id, input.startedAt); - for (const runId of issueRuns) observedRunIds.add(runId); - - const sortedRunIds = [...observedRunIds]; - if (!failureStage && latestIssue.status !== "done") { - failureStage = mapIssueStatusToFailureStage(latestIssue.status, finalRun?.status ?? null); - } - - let extractedPatchPath: string | undefined; - try { - extractedPatchPath = await extractIssuePatch({ - issue: latestIssue, - outputDir: input.outputDir, - taskDir: input.taskDir, - }); - if (latestIssue.currentExecutionWorkspace?.mode === "shared_workspace") { - notes = appendNote( - notes, - `Patch was extracted from shared workspace ${latestIssue.currentExecutionWorkspace.name}; unrelated dirty changes in that workspace can contaminate the diff.`, - ); - } - if (!extractedPatchPath) { - failureStage = failureStage ?? "patch_missing"; - notes = appendNote(notes, `No git diff was available in the execution workspace for issue ${latestIssue.identifier ?? latestIssue.id}.`); - } - } catch (err) { - failureStage = failureStage ?? "patch_extract"; - notes = appendNote(notes, err instanceof Error ? err.message : String(err)); - } - - return { - issue: latestIssue, - runIds: sortedRunIds, - status: latestIssue.status, - patchPath: extractedPatchPath, - failureStage, - notes, - }; - }; -} - -function parseJsonlObjects(contents: string, filePath: string): Array> { - const rows: Array> = []; - const lines = contents.split(/\r?\n/); - - lines.forEach((line, index) => { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith("#")) return; - - let parsed: unknown; - try { - parsed = JSON.parse(trimmed); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - throw new Error(`${filePath}:${index + 1}: invalid JSONL row: ${message}`); - } - - if (!isRecord(parsed)) { - throw new Error(`${filePath}:${index + 1}: expected a JSON object`); - } - rows.push(parsed); - }); - - if (rows.length === 0) { - throw new Error(`${filePath}: no benchmark tasks found`); - } - - return rows; -} - -function normalizeRepoSpec(raw: Record): BenchRepoSpec | undefined { - const rawRepo = raw.repo; - const startingCommit = firstString(raw, ["startingCommit", "starting_commit", "baseCommit", "base_commit"]); - - if (typeof rawRepo === "string" && rawRepo.trim()) { - return compactRepoSpec({ url: rawRepo.trim(), baseCommit: startingCommit }); - } - - if (isRecord(rawRepo)) { - return compactRepoSpec({ - url: firstString(rawRepo, ["url", "remote", "repository"]), - path: firstString(rawRepo, ["path", "localPath", "local_path"]), - baseCommit: firstString(rawRepo, ["baseCommit", "base_commit", "startingCommit", "starting_commit"]) ?? startingCommit, - }); - } - - return startingCommit ? { baseCommit: startingCommit } : undefined; -} - -function normalizeIssueRef(raw: Record): string | undefined { - const metadata = asRecord(raw.metadata); - return firstString(raw, [ - "paperclipIssueRef", - "paperclip_issue_ref", - "paperclipIssueId", - "paperclip_issue_id", - "paperclipIssueIdentifier", - "paperclip_issue_identifier", - "existingIssueId", - "existing_issue_id", - "issueRef", - "issue_ref", - ]) ?? (metadata - ? firstString(metadata, [ - "paperclipIssueRef", - "paperclipIssueId", - "paperclipIssueIdentifier", - "existingIssueId", - "issueRef", - ]) - : undefined); -} - -function compactRepoSpec(repo: BenchRepoSpec): BenchRepoSpec | undefined { - const compact = { - url: repo.url?.trim() || undefined, - path: repo.path?.trim() || undefined, - baseCommit: repo.baseCommit?.trim() || undefined, - }; - return compact.url || compact.path || compact.baseCommit ? compact : undefined; -} - -function normalizeRubric(value: unknown): BenchRubricItem[] { - if (Array.isArray(value)) { - return value.map((item, index) => { - if (typeof item === "string" && item.trim()) { - return { name: item.trim() }; - } - if (isRecord(item)) { - const name = firstString(item, ["name", "criterion", "label"]); - if (!name) throw new Error(`human_rubric[${index}] requires name`); - return { - name, - description: firstString(item, ["description", "notes"]), - maxScore: firstNumber(item, ["maxScore", "max_score", "points"]), - }; - } - throw new Error(`human_rubric[${index}] must be a string or object`); - }); - } - - if (isRecord(value)) { - return Object.entries(value).map(([name, raw]) => { - if (typeof raw === "number") return { name, maxScore: raw }; - if (typeof raw === "string") return { name, description: raw }; - if (isRecord(raw)) { - return { - name, - description: firstString(raw, ["description", "notes"]), - maxScore: firstNumber(raw, ["maxScore", "max_score", "points"]), - }; - } - return { name }; - }); - } - - return []; -} - -function checkoutTaskRepo(task: BenchTask, workspacePath: string): void { - const source = task.repo?.url ?? task.repo?.path; - if (!source) throw new Error(`Task ${task.id} has no repository source`); - - const clone = spawnSync("git", ["clone", "--no-tags", source, workspacePath], { - encoding: "utf8", - }); - if (clone.status !== 0) { - throw new Error(`git clone failed for task ${task.id}: ${clone.stderr || clone.stdout}`); - } - - if (task.repo?.baseCommit) { - const checkout = spawnSync("git", ["checkout", task.repo.baseCommit], { - cwd: workspacePath, - encoding: "utf8", - }); - if (checkout.status !== 0) { - throw new Error(`git checkout failed for task ${task.id}: ${checkout.stderr || checkout.stdout}`); - } - } -} - -async function writeTrace( - tracePath: string, - taskId: string, - startedAt: string, - status: BenchResult["status"], - events: Array>, - error?: string, - extra?: Record, -): Promise { - await writeFile( - tracePath, - JSON.stringify( - { - schemaVersion: 1, - taskId, - startedAt, - endedAt: new Date().toISOString(), - status, - error, - events, - ...(extra ?? {}), - }, - null, - 2, - ), - "utf8", - ); -} - -function readWakeupRunId(response: AgentWakeupResponse | null): string | null { - if (!response) return null; - if ("id" in response && typeof response.id === "string") { - return response.id; - } - if ("executionRunId" in response && typeof response.executionRunId === "string") { - return response.executionRunId; - } - return null; -} - -function readWakeupNotes(response: AgentWakeupResponse | null): string | undefined { - if (!response || !("status" in response) || response.status !== "skipped") return undefined; - const parts = [response.reason, response.message].filter((value): value is string => Boolean(value && value.trim())); - return parts.length > 0 ? `Heartbeat wakeup skipped: ${parts.join(" - ")}` : undefined; -} - -function resolveWallClockBudgetMs(task: BenchTask, defaultMaxWallClockMinutes?: number): number | undefined { - const minutes = task.maxWallClockMinutes ?? defaultMaxWallClockMinutes; - if (minutes === undefined) return undefined; - return Math.max(1, Math.ceil(minutes * 60_000)); -} - -function readHeartbeatRunCostUsd(run: HeartbeatRun | null | undefined): number | null { - const usage = asRecord(run?.usageJson); - const result = asRecord(run?.resultJson); - return firstFiniteNumber( - usage ? [usage.costUsd, usage.cost_usd, usage.total_cost_usd] : [], - result ? [result.costUsd, result.cost_usd, result.total_cost_usd] : [], - ); -} - -async function listIssueRunIdsSince( - ctx: ReturnType, - companyId: string, - agentId: string, - issueId: string, - startedAt: string, -): Promise { - const runs = await ctx.api.get(`/api/companies/${companyId}/heartbeat-runs?agentId=${agentId}&limit=200`); - if (!Array.isArray(runs)) return []; - const startedAtMs = toTimestamp(startedAt) ?? 0; - return runs - .filter((run) => { - const context = asRecord(run.contextSnapshot); - return ( - typeof run.id === "string" - && (toTimestamp(run.createdAt) ?? 0) >= startedAtMs - && context?.issueId === issueId - ); - }) - .sort((left, right) => (toTimestamp(left.createdAt) ?? 0) - (toTimestamp(right.createdAt) ?? 0)) - .map((run) => run.id); -} - -async function extractIssuePatch(input: { - issue: Issue; - outputDir: string; - taskDir: string; -}): Promise { - const workspaceCwd = input.issue.currentExecutionWorkspace?.cwd; - if (!workspaceCwd) { - throw new BenchStageError( - "patch_extract", - `Issue ${input.issue.identifier ?? input.issue.id} does not have a local execution workspace available for patch extraction`, - ); - } - - const statusBefore = spawnSync("git", ["-C", workspaceCwd, "status", "--porcelain=v1"], { - encoding: "utf8", - }); - if (statusBefore.status !== 0) { - throw new BenchStageError( - "patch_extract", - `git status failed for ${workspaceCwd}: ${statusBefore.stderr || statusBefore.stdout}`, - ); - } - - const untrackedFiles = statusBefore.stdout - .split(/\r?\n/) - .filter((line) => line.startsWith("?? ")) - .map((line) => line.slice(3).trim()) - .filter(Boolean); - if (untrackedFiles.length > 0) { - const addIntent = spawnSync("git", ["-C", workspaceCwd, "add", "-N", "--", ...untrackedFiles], { - encoding: "utf8", - }); - if (addIntent.status !== 0) { - throw new BenchStageError( - "patch_extract", - `git add -N failed for ${workspaceCwd}: ${addIntent.stderr || addIntent.stdout}`, - ); - } - } - - const diff = spawnSync("git", ["-C", workspaceCwd, "diff", "--binary", "--no-ext-diff", "--submodule=diff", "HEAD", "--"], { - encoding: "utf8", - maxBuffer: 16 * 1024 * 1024, - }); - if (diff.status !== 0) { - throw new BenchStageError( - "patch_extract", - `git diff failed for ${workspaceCwd}: ${diff.stderr || diff.stdout}`, - ); - } - if (!diff.stdout.trim()) return undefined; - - const patchFilePath = path.join(input.taskDir, "patch.diff"); - await writeFile(patchFilePath, diff.stdout, "utf8"); - return path.relative(input.outputDir, patchFilePath); -} - -function mapIssueStatusToFailureStage( - status: Issue["status"], - runStatus: HeartbeatRun["status"] | null, -): string | null { - if (status === "done") return null; - if (status === "blocked") return "issue_blocked"; - if (status === "cancelled") return "issue_cancelled"; - if (status === "in_review") return "issue_in_review"; - if (runStatus === "timed_out") return "run_timeout"; - if (runStatus === "cancelled") return "run_cancelled"; - if (runStatus === "failed") return "run_failed"; - return "issue_incomplete"; -} - -function appendNote(existing: string | undefined, addition: string): string { - const trimmed = addition.trim(); - if (!trimmed) return existing ?? ""; - if (!existing?.trim()) return trimmed; - return `${existing}\n${trimmed}`; -} - -function toTimestamp(value: Date | string | null | undefined): number | null { - if (!value) return null; - const timestamp = value instanceof Date ? value.getTime() : new Date(value).getTime(); - return Number.isFinite(timestamp) ? timestamp : null; -} - -function firstFiniteNumber(...groups: unknown[][]): number | null { - for (const group of groups) { - for (const value of group) { - if (typeof value === "number" && Number.isFinite(value)) return value; - if (typeof value === "string" && value.trim()) { - const parsed = Number(value); - if (Number.isFinite(parsed)) return parsed; - } - } - } - return null; -} - -function firstString(record: Record, keys: string[]): string | undefined { - for (const key of keys) { - const value = record[key]; - if (typeof value === "string" && value.trim()) return value.trim(); - } - return undefined; -} - -function firstNumber(record: Record, keys: string[]): number | undefined { - for (const key of keys) { - const value = record[key]; - if (typeof value === "number" && Number.isFinite(value)) return value; - if (typeof value === "string" && value.trim()) { - const parsed = Number(value); - if (Number.isFinite(parsed)) return parsed; - } - } - return undefined; -} - -function firstStringArray(record: Record, keys: string[]): string[] { - for (const key of keys) { - const value = record[key]; - if (Array.isArray(value)) { - return value - .map((item) => typeof item === "string" ? item.trim() : "") - .filter(Boolean); - } - } - return []; -} - -function asRecord(value: unknown): Record | undefined { - return isRecord(value) ? value : undefined; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function parseOptionalPositiveNumber(value: string | undefined, label: string): number | undefined { - if (value === undefined || value.trim() === "") return undefined; - const parsed = Number(value); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error(`--${label} must be a positive number`); - } - return parsed; -} - -function parseOptionalPositiveInteger(value: string | undefined, label: string): number | undefined { - const parsed = parseOptionalPositiveNumber(value, label); - if (parsed === undefined) return undefined; - if (!Number.isInteger(parsed)) { - throw new Error(`--${label} must be a positive integer`); - } - return parsed; -} - -function normalizeIdentifier(value: string, label: string): string { - const trimmed = value.trim(); - if (!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(trimmed)) { - throw new Error(`${label} must start with an alphanumeric character and contain only letters, numbers, '.', '_', ':', or '-'`); - } - return trimmed; -} - -function safePathSegment(value: string): string { - return value.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "") || "task"; -} - -function buildDefaultRunId(benchmark: string): string { - const stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z"); - return `${stamp}-${safePathSegment(benchmark)}`; -} diff --git a/cli/src/index.ts b/cli/src/index.ts index 53867bebccb..bbec356f342 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -18,7 +18,6 @@ import { registerActivityCommands } from "./commands/client/activity.js"; import { registerDashboardCommands } from "./commands/client/dashboard.js"; import { registerRoutineCommands } from "./commands/routines.js"; import { registerFeedbackCommands } from "./commands/client/feedback.js"; -import { registerBenchCommands } from "./commands/bench.js"; import { applyDataDirOverride, type DataDirOptionLike } from "./config/data-dir.js"; import { loadPaperclipEnvFile } from "./config/env.js"; import { initTelemetryFromConfigFile, flushTelemetry } from "./telemetry.js"; @@ -148,7 +147,6 @@ registerActivityCommands(program); registerDashboardCommands(program); registerRoutineCommands(program); registerFeedbackCommands(program); -registerBenchCommands(program); registerWorktreeCommands(program); registerEnvLabCommands(program); registerPluginCommands(program); diff --git a/evals/README.md b/evals/README.md index efc183973ce..5974d98cfe1 100644 --- a/evals/README.md +++ b/evals/README.md @@ -34,53 +34,6 @@ promptfoo eval promptfoo view ``` -## Paperclip Bench - -`paperclipai bench` is the first-party benchmark harness for Paperclip-native and SWE-style task suites. It supports a deterministic `prepare` runner for prompt/render artifacts and a live `paperclip` runner that creates or reuses benchmark issues, wakes the assignee, and captures run metadata plus a final patch artifact. - -```bash -# Prepare the bundled smoke suite -pnpm bench:smoke - -# Validate any JSONL suite -pnpm paperclipai bench validate --tasks evals/bench/tasks/paperclip-native-smoke.jsonl - -# Prepare artifacts for a custom suite -pnpm paperclipai bench run \ - --tasks path/to/tasks.jsonl \ - --benchmark missionbench \ - --paperclip-version current \ - --model anthropic/claude-sonnet-4.5 \ - --max-cost-usd 10 \ - --max-wall-clock-minutes 120 \ - --output .tmp/paperclip-bench/missionbench-run - -# Run tasks live against a Paperclip dev server -pnpm paperclipai bench run \ - --runner paperclip \ - --tasks path/to/tasks.jsonl \ - --benchmark missionbench \ - --paperclip-version current \ - --model anthropic/claude-sonnet-4.5 \ - --company-id \ - --assignee-agent-id \ - --project-id \ - --goal-id \ - --max-cost-usd 10 \ - --max-wall-clock-minutes 120 \ - --output .tmp/paperclip-bench/missionbench-live -``` - -Task rows are JSONL objects. The parser accepts both camelCase and snake_case fields so suites can use MissionBench-style records: - -```json -{"id":"missionbench-001","title":"Add audit logs","repo":"github.com/paperclipai/missionbench-saas","starting_commit":"abc123","problem_statement":"Add organization-level audit logs.","success_checks":["npm test"],"required_artifacts":["implementation patch","final PR summary"]} -``` - -Set `paperclip_issue_id`, `paperclip_issue_identifier`, `paperclipIssueRef`, or the same keys inside `metadata` to reuse an existing benchmark issue from the task row. Otherwise, `--create-issues -C ` creates one Paperclip issue per task for the `prepare` runner, and the live `paperclip` runner creates issues automatically when the task row does not already reference one. Created issues use the normalized prompt as the issue description and set `billingCode` to `bench:` unless `--billing-code` is provided. - -Live-run `results.jsonl` and per-task `trace.json` include the Paperclip issue id, observed heartbeat run ids, the final task status, a relative `patchPath` when a git diff was extracted from the execution workspace, and a `failureStage` when the task stops short of a clean `done` result. - ### What's tested Phase 0 covers narrow behavior evals for the Paperclip heartbeat skill: diff --git a/evals/bench/.gitignore b/evals/bench/.gitignore deleted file mode 100644 index 23f7ddb4808..00000000000 --- a/evals/bench/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -runs/* -!runs/.gitkeep -!runs/README.md diff --git a/evals/bench/README.md b/evals/bench/README.md deleted file mode 100644 index 579c4888c11..00000000000 --- a/evals/bench/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# paperclip-bench - -`paperclip-bench` is the scaffold for Paperclip's benchmark harness and benchmark content. Phase 1 keeps it inside the monorepo under `evals/bench/`, but the layout is intentionally shaped so it can move into a standalone repository later with minimal churn. - -This phase does not implement the runner. It establishes: - -- where imported benchmark manifests belong -- where Paperclip-native tasks belong -- where starter company packages belong -- where runner, evaluator, Python bridge, ops, and generated run artifacts belong -- the initial schema contract for task and suite manifests - -## Layout - -| Path | Purpose | -|---|---| -| `bench.schema.json` | Shared JSON Schema definitions for `paperclip-bench/task/v1` and `paperclip-bench/suite/v1`. | -| `suites/` | Versioned suite manifests such as local smoke, SWE-bench Lite smoke, and Paperclip-native smoke. | -| `benchmarks/imported/` | Imported benchmark metadata and normalized task manifests without vendoring third-party source repositories. | -| `benchmarks/paperclip-native/` | Paperclip-native tasks, prompts, rubrics, evaluator config, and hidden-test placeholders. | -| `companies/` | Starter company packages used to seed Paperclip-native benchmark runs. | -| `fixtures/` | Fixture repo placeholders plus shared env/secrets examples. | -| `src/` | Future TypeScript CLI, task normalization, runners, evaluators, metrics, and reports. Placeholder subdirectories are checked in so contributors can see the intended module boundaries. | -| `python/` | Future bridge scripts for ecosystems that already expect Python entrypoints. Placeholder subdirectories reserve Python bridge, evaluator, and dataset helpers. | -| `ops/` | Local Docker and single-machine cloud execution scaffolding, plus future CI entrypoints. | -| `runs/` | Generated benchmark outputs. Raw runs stay untracked by default, with only a placeholder kept in git. | - -## Manifest Validation - -Phase 1 picks JSON Schema plus YAML manifests. The validation script loads YAML, validates against `bench.schema.json`, and checks both suite and task examples: - -```bash -node evals/bench/scripts/validate-manifests.mjs -# or -pnpm --dir evals/bench run validate:manifests -``` - -## V0 Verification Intent - -The first vertical slice will be considered wired correctly when these commands are all true: - -1. `node evals/bench/scripts/validate-manifests.mjs` passes. -2. A contributor can locate imported tasks, native tasks, starter companies, runner code, Python bridge code, ops files, and generated runs from this directory layout alone. -3. Later phases can add implementation in place without changing manifest ids or top-level directory meaning. - -## Notes - -- `.paperclip.yaml` starter-company files are placeholders in this phase. Their exact import/runtime contract lands with the Paperclip runner integration work. -- Hidden tests are represented as directories only. Real evaluator wiring lands in later phases. -- The current Paperclip CLI already has early bench commands under `cli/src/commands/bench.ts`; this scaffold defines the repository contract those commands can grow into. diff --git a/evals/bench/bench.schema.json b/evals/bench/bench.schema.json deleted file mode 100644 index a5c21be75e8..00000000000 --- a/evals/bench/bench.schema.json +++ /dev/null @@ -1,272 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://paperclip.ai/schemas/paperclip-bench/bench.schema.json", - "title": "paperclip-bench manifest schema", - "$defs": { - "pathRef": { - "type": "string", - "minLength": 1 - }, - "runnerKind": { - "type": "string", - "enum": ["paperclip", "direct-agent", "noop", "prepare"] - }, - "repo": { - "type": "object", - "additionalProperties": false, - "required": ["kind", "url", "ref"], - "properties": { - "kind": { - "type": "string", - "enum": ["git", "local-fixture"] - }, - "url": { - "type": "string", - "minLength": 1 - }, - "ref": { - "type": "string", - "minLength": 1 - }, - "workdir": { - "type": "string", - "minLength": 1 - } - } - }, - "budget": { - "type": "object", - "additionalProperties": false, - "properties": { - "max_wall_clock_minutes": { - "type": "integer", - "minimum": 1 - }, - "max_cost_usd": { - "type": "number", - "minimum": 0 - }, - "max_tool_calls": { - "type": "integer", - "minimum": 1 - }, - "max_test_runs": { - "type": "integer", - "minimum": 1 - } - } - }, - "stop": { - "type": "object", - "additionalProperties": false, - "properties": { - "stable_idle_minutes": { - "type": "integer", - "minimum": 0 - }, - "require_terminal_issue_state": { - "type": "boolean" - }, - "allow_human_intervention": { - "type": "boolean" - } - } - }, - "runner": { - "type": "object", - "additionalProperties": false, - "required": ["kind"], - "properties": { - "kind": { - "$ref": "#/$defs/runnerKind" - }, - "mode": { - "type": "string", - "minLength": 1 - } - } - }, - "evaluator": { - "type": "object", - "additionalProperties": false, - "required": ["kind"], - "properties": { - "kind": { - "type": "string", - "enum": ["command", "swebench", "swebench-pro", "paperclip-native"] - }, - "manifest": { - "$ref": "#/$defs/pathRef" - }, - "prediction_format": { - "type": "string", - "minLength": 1 - }, - "visible_commands": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - }, - "hidden_commands": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - } - }, - "taskV1": { - "type": "object", - "additionalProperties": false, - "required": ["schema", "id", "family", "summary", "repo", "runner", "evaluator"], - "properties": { - "$schema": { - "type": "string", - "minLength": 1 - }, - "schema": { - "type": "string", - "const": "paperclip-bench/task/v1" - }, - "id": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9._:-]*$" - }, - "family": { - "type": "string", - "minLength": 1 - }, - "summary": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string" - }, - "source": { - "type": "object" - }, - "repo": { - "$ref": "#/$defs/repo" - }, - "prompt": { - "type": "object", - "additionalProperties": false, - "properties": { - "path": { - "$ref": "#/$defs/pathRef" - } - } - }, - "starter_company": { - "type": "object", - "additionalProperties": false, - "properties": { - "package": { - "$ref": "#/$defs/pathRef" - } - } - }, - "runner": { - "$ref": "#/$defs/runner" - }, - "budget": { - "$ref": "#/$defs/budget" - }, - "stop": { - "$ref": "#/$defs/stop" - }, - "evaluator": { - "$ref": "#/$defs/evaluator" - }, - "required_artifacts": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - }, - "metrics_profile": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - }, - "tags": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - } - }, - "suiteV1": { - "type": "object", - "additionalProperties": false, - "required": ["schema", "id", "title", "tasks"], - "properties": { - "$schema": { - "type": "string", - "minLength": 1 - }, - "schema": { - "type": "string", - "const": "paperclip-bench/suite/v1" - }, - "id": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9._-]*$" - }, - "title": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string" - }, - "runner": { - "$ref": "#/$defs/runner" - }, - "paperclip_version": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "tasks": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/$defs/pathRef" - } - }, - "defaults": { - "type": "object", - "additionalProperties": false, - "properties": { - "budget": { - "$ref": "#/$defs/budget" - }, - "stop": { - "$ref": "#/$defs/stop" - } - } - }, - "notes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - } - } - } -} diff --git a/evals/bench/benchmarks/imported/swebench-lite/README.md b/evals/bench/benchmarks/imported/swebench-lite/README.md deleted file mode 100644 index dff75435a51..00000000000 --- a/evals/bench/benchmarks/imported/swebench-lite/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# SWE-bench Lite Imports - -This directory stores normalized task manifests and dataset notes for imported SWE-bench Lite tasks. It does not vendor upstream benchmark repositories or evaluator code. - -Use `tasks/` for one manifest per selected instance. Keep raw upstream metadata under `source` so later bridge code can emit official prediction formats without guessing. diff --git a/evals/bench/benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml b/evals/bench/benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml deleted file mode 100644 index 84571435dde..00000000000 --- a/evals/bench/benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml +++ /dev/null @@ -1,39 +0,0 @@ -$schema: ../../../../bench.schema.json#/$defs/taskV1 -schema: paperclip-bench/task/v1 -id: swebench-lite:pytest-dev__pytest-6116 -family: swebench-lite -summary: Fix a small pytest CLI behavior regression imported from SWE-bench Lite. -description: Imported smoke task used to validate the scaffold and preserve upstream task metadata. -source: - dataset: SWE-bench/SWE-bench_Lite - instance_id: pytest-dev__pytest-6116 - repo: pytest-dev/pytest - base_commit: 17f7b44f2f1b8b1a6b2cb67f7e6f01a8a5b9ea12 - version: "6.0" -repo: - kind: git - url: https://github.com/pytest-dev/pytest.git - ref: 17f7b44f2f1b8b1a6b2cb67f7e6f01a8a5b9ea12 -runner: - kind: prepare - mode: isolated-instance -budget: - max_wall_clock_minutes: 60 - max_cost_usd: 5 - max_tool_calls: 150 -stop: - stable_idle_minutes: 5 - require_terminal_issue_state: true -evaluator: - kind: swebench - prediction_format: swebench-jsonl -required_artifacts: - - prediction.patch - - results.json -metrics_profile: - - patch - - cost - - evaluator -tags: - - imported - - smoke diff --git a/evals/bench/benchmarks/paperclip-native/README.md b/evals/bench/benchmarks/paperclip-native/README.md deleted file mode 100644 index 7d0bb9d2f91..00000000000 --- a/evals/bench/benchmarks/paperclip-native/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Paperclip-Native Tasks - -Paperclip-native tasks test the control plane itself: decomposition, blockers, handoffs, artifact quality, liveness, and reviewability. Each task owns its prompt, rubric, evaluator description, and hidden-test placeholders in one directory. diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/evaluator.yaml b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/evaluator.yaml deleted file mode 100644 index ea0d6cb3fbd..00000000000 --- a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/evaluator.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: command -visible_commands: - - pnpm test - - pnpm -r typecheck -hidden_commands: - - pnpm test:e2e:audit-log -notes: - - Hidden tests stay outside the agent workspace once evaluator wiring lands. diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/hidden-tests/README.md b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/hidden-tests/README.md deleted file mode 100644 index fcfc738a2b3..00000000000 --- a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/hidden-tests/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Hidden Tests Placeholder - -Real hidden tests are intentionally omitted in Phase 1. This directory exists so later phases have a stable location for evaluator-only assets. diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/issue.md b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/issue.md deleted file mode 100644 index 1cd08b7c443..00000000000 --- a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/issue.md +++ /dev/null @@ -1,10 +0,0 @@ -# Add Organization Audit Logs - -Implement organization-level audit logs for account settings changes in the demo SaaS fixture. - -Definition of done: - -- account settings mutations create durable audit-log records -- the app exposes enough UI or API surface for operators to inspect the new entries -- relevant tests are updated or added -- the final summary names what changed, what was verified, and remaining risks diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/rubric.md b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/rubric.md deleted file mode 100644 index 1dad3f60120..00000000000 --- a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/rubric.md +++ /dev/null @@ -1,9 +0,0 @@ -# Rubric - -Score from 0-5 on: - -- architecture -- minimality -- test quality -- final summary quality -- operational reviewability diff --git a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml b/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml deleted file mode 100644 index 8ca3f75e57d..00000000000 --- a/evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml +++ /dev/null @@ -1,49 +0,0 @@ -$schema: ../../../../bench.schema.json#/$defs/taskV1 -schema: paperclip-bench/task/v1 -id: pclip-native-001-audit-log-mission -family: paperclip-native -summary: Add organization-level audit logs to a demo SaaS app through a small engineering squad. -description: Native smoke task focused on implementation, verification, and final review artifacts. -repo: - kind: local-fixture - url: ../../../../fixtures/repos/demo-saas - ref: scaffold - workdir: . -prompt: - path: ./issue.md -starter_company: - package: ../../../../companies/eng-squad-basic -runner: - kind: paperclip - mode: isolated-instance -budget: - max_wall_clock_minutes: 120 - max_cost_usd: 10 - max_tool_calls: 300 - max_test_runs: 20 -stop: - stable_idle_minutes: 5 - require_terminal_issue_state: true - allow_human_intervention: false -evaluator: - kind: command - manifest: ./evaluator.yaml - visible_commands: - - pnpm test - - pnpm -r typecheck - hidden_commands: - - pnpm test:e2e:audit-log -required_artifacts: - - prediction.patch - - final_summary.md - - tests_run.md - - risks.md -metrics_profile: - - patch - - cost - - paperclip-control-plane - - liveness - - reviewability -tags: - - native - - smoke diff --git a/evals/bench/companies/README.md b/evals/bench/companies/README.md deleted file mode 100644 index 13af4455a43..00000000000 --- a/evals/bench/companies/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Starter Companies - -Starter company packages seed Paperclip-native tasks with a known initial org state. These directories follow the Agent Companies markdown package shape and add a provisional `.paperclip.yaml` file for benchmark-specific hints. diff --git a/evals/bench/companies/ceo-only/.paperclip.yaml b/evals/bench/companies/ceo-only/.paperclip.yaml deleted file mode 100644 index bac45c2f6bb..00000000000 --- a/evals/bench/companies/ceo-only/.paperclip.yaml +++ /dev/null @@ -1,5 +0,0 @@ -version: 1 -company_name: CEO Only -benchmark: - enable_agent_timers: false - intended_runner: paperclip diff --git a/evals/bench/companies/ceo-only/COMPANY.md b/evals/bench/companies/ceo-only/COMPANY.md deleted file mode 100644 index 75763fea955..00000000000 --- a/evals/bench/companies/ceo-only/COMPANY.md +++ /dev/null @@ -1,7 +0,0 @@ -# CEO Only - -Purpose: minimal bootstrap company for imported or direct-agent style tasks. - -Roles: - -- CEO owns the single benchmark issue. diff --git a/evals/bench/companies/ceo-only/agents/ceo/AGENTS.md b/evals/bench/companies/ceo-only/agents/ceo/AGENTS.md deleted file mode 100644 index 2153dd61f46..00000000000 --- a/evals/bench/companies/ceo-only/agents/ceo/AGENTS.md +++ /dev/null @@ -1,9 +0,0 @@ -# CEO - -You own the benchmark issue directly. - -Default expectations: - -- read the task prompt -- do the work or delegate when the harness allows it -- finish with reviewable artifacts diff --git a/evals/bench/companies/eng-squad-basic/.paperclip.yaml b/evals/bench/companies/eng-squad-basic/.paperclip.yaml deleted file mode 100644 index 9342ba1dbdf..00000000000 --- a/evals/bench/companies/eng-squad-basic/.paperclip.yaml +++ /dev/null @@ -1,6 +0,0 @@ -version: 1 -company_name: Engineering Squad Basic -benchmark: - enable_agent_timers: false - intended_runner: paperclip - default_idle_stability_minutes: 5 diff --git a/evals/bench/companies/eng-squad-basic/COMPANY.md b/evals/bench/companies/eng-squad-basic/COMPANY.md deleted file mode 100644 index 4967e170632..00000000000 --- a/evals/bench/companies/eng-squad-basic/COMPANY.md +++ /dev/null @@ -1,10 +0,0 @@ -# Engineering Squad Basic - -Purpose: small delivery-focused company package for Paperclip-native smoke tasks. - -Roles: - -- CEO for coordination -- CTO for task routing and technical review -- Senior Engineer for implementation -- QA for verification diff --git a/evals/bench/companies/eng-squad-basic/agents/ceo/AGENTS.md b/evals/bench/companies/eng-squad-basic/agents/ceo/AGENTS.md deleted file mode 100644 index 2b226bd924f..00000000000 --- a/evals/bench/companies/eng-squad-basic/agents/ceo/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# CEO - -Coordinate the benchmark mission, keep work moving, and ensure the final artifact set is reviewable. diff --git a/evals/bench/companies/eng-squad-basic/agents/cto/AGENTS.md b/evals/bench/companies/eng-squad-basic/agents/cto/AGENTS.md deleted file mode 100644 index 528f7b2f759..00000000000 --- a/evals/bench/companies/eng-squad-basic/agents/cto/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# CTO - -Break the mission into concrete engineering work, keep technical reviews crisp, and avoid unnecessary task churn. diff --git a/evals/bench/companies/eng-squad-basic/agents/qa/AGENTS.md b/evals/bench/companies/eng-squad-basic/agents/qa/AGENTS.md deleted file mode 100644 index 324dfc135ef..00000000000 --- a/evals/bench/companies/eng-squad-basic/agents/qa/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# QA - -Validate the deliverable against visible checks, inspect the final summary, and report residual risks. diff --git a/evals/bench/companies/eng-squad-basic/agents/senior-engineer/AGENTS.md b/evals/bench/companies/eng-squad-basic/agents/senior-engineer/AGENTS.md deleted file mode 100644 index 6c34ab3d9f6..00000000000 --- a/evals/bench/companies/eng-squad-basic/agents/senior-engineer/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# Senior Engineer - -Implement the requested change, run relevant validation, and leave clear handoff notes for review. diff --git a/evals/bench/companies/governed-eng-squad/.paperclip.yaml b/evals/bench/companies/governed-eng-squad/.paperclip.yaml deleted file mode 100644 index fe793e97bdd..00000000000 --- a/evals/bench/companies/governed-eng-squad/.paperclip.yaml +++ /dev/null @@ -1,6 +0,0 @@ -version: 1 -company_name: Governed Engineering Squad -benchmark: - enable_agent_timers: false - intended_runner: paperclip - requires_review_handoffs: true diff --git a/evals/bench/companies/governed-eng-squad/COMPANY.md b/evals/bench/companies/governed-eng-squad/COMPANY.md deleted file mode 100644 index 1de6b54cb9b..00000000000 --- a/evals/bench/companies/governed-eng-squad/COMPANY.md +++ /dev/null @@ -1,12 +0,0 @@ -# Governed Engineering Squad - -Purpose: review-heavy starter package for blocker and handoff benchmarks. - -Roles: - -- CEO -- CTO -- Senior Engineer -- Security Engineer -- UX Designer -- QA diff --git a/evals/bench/companies/governed-eng-squad/agents/ceo/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/ceo/AGENTS.md deleted file mode 100644 index d80943b1cf0..00000000000 --- a/evals/bench/companies/governed-eng-squad/agents/ceo/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# CEO - -Own the benchmark outcome, keep review dependencies explicit, and avoid hidden state. diff --git a/evals/bench/companies/governed-eng-squad/agents/cto/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/cto/AGENTS.md deleted file mode 100644 index ac431e5b207..00000000000 --- a/evals/bench/companies/governed-eng-squad/agents/cto/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# CTO - -Manage implementation sequencing, review dependencies, and blocker hygiene. diff --git a/evals/bench/companies/governed-eng-squad/agents/qa/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/qa/AGENTS.md deleted file mode 100644 index 01a2e545a75..00000000000 --- a/evals/bench/companies/governed-eng-squad/agents/qa/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# QA - -Validate acceptance criteria, inspect artifacts, and report whether the mission is ready for review. diff --git a/evals/bench/companies/governed-eng-squad/agents/security-engineer/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/security-engineer/AGENTS.md deleted file mode 100644 index 095861a673a..00000000000 --- a/evals/bench/companies/governed-eng-squad/agents/security-engineer/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# Security Engineer - -Review security-sensitive changes, document required controls, and block work explicitly when requirements are incomplete. diff --git a/evals/bench/companies/governed-eng-squad/agents/senior-engineer/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/senior-engineer/AGENTS.md deleted file mode 100644 index 0ac44ad08f1..00000000000 --- a/evals/bench/companies/governed-eng-squad/agents/senior-engineer/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# Senior Engineer - -Implement the technical work after required reviews are in place and leave evidence for QA. diff --git a/evals/bench/companies/governed-eng-squad/agents/ux-designer/AGENTS.md b/evals/bench/companies/governed-eng-squad/agents/ux-designer/AGENTS.md deleted file mode 100644 index a10d3eb85a6..00000000000 --- a/evals/bench/companies/governed-eng-squad/agents/ux-designer/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# UX Designer - -Review operator-facing states and copy so benchmark tasks can measure design-review handoffs. diff --git a/evals/bench/fixtures/README.md b/evals/bench/fixtures/README.md deleted file mode 100644 index eb3c43e6df7..00000000000 --- a/evals/bench/fixtures/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Fixtures - -Fixture repos and shared env examples for benchmark tasks live here. Real fixture repositories land in later phases; Phase 1 keeps stable placeholders so tasks can reference durable paths. diff --git a/evals/bench/fixtures/repos/demo-saas/.gitkeep b/evals/bench/fixtures/repos/demo-saas/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/fixtures/repos/demo-saas/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/fixtures/repos/workflow-demo/.gitkeep b/evals/bench/fixtures/repos/workflow-demo/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/fixtures/repos/workflow-demo/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/fixtures/secrets.example.env b/evals/bench/fixtures/secrets.example.env deleted file mode 100644 index 0a4b8a3c50f..00000000000 --- a/evals/bench/fixtures/secrets.example.env +++ /dev/null @@ -1,5 +0,0 @@ -# Example env file for local benchmark runs. -# Real values should be injected per run, not committed. -OPENAI_API_KEY= -ANTHROPIC_API_KEY= -PAPERCLIP_API_URL=http://localhost:3100 diff --git a/evals/bench/ops/README.md b/evals/bench/ops/README.md deleted file mode 100644 index 99ae6127248..00000000000 --- a/evals/bench/ops/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Ops Layout - -Operational scaffolding for benchmark execution lives here. - -- `docker/`: local container images and compose files for reproducible runs -- `single-machine-cloud/`: scripts and templates for one-box hosted execution -- `ci/`: future CI entrypoints and release plumbing for benchmark automation diff --git a/evals/bench/ops/ci/.gitkeep b/evals/bench/ops/ci/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/ops/ci/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/ops/docker/.gitkeep b/evals/bench/ops/docker/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/ops/docker/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/ops/single-machine-cloud/.gitkeep b/evals/bench/ops/single-machine-cloud/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/ops/single-machine-cloud/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/package.json b/evals/bench/package.json deleted file mode 100644 index bfa6e4a0c8a..00000000000 --- a/evals/bench/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "paperclip-bench", - "private": true, - "version": "0.0.0", - "type": "module", - "description": "Scaffold for Paperclip benchmark suites, tasks, and runner assets.", - "scripts": { - "validate:manifests": "node ./scripts/validate-manifests.mjs" - }, - "devDependencies": { - "ajv": "^8.17.1", - "ajv-formats": "^3.0.1", - "yaml": "^2.8.1" - }, - "packageManager": "pnpm@9.15.4" -} diff --git a/evals/bench/pyproject.toml b/evals/bench/pyproject.toml deleted file mode 100644 index 4b1bc34906d..00000000000 --- a/evals/bench/pyproject.toml +++ /dev/null @@ -1,11 +0,0 @@ -[project] -name = "paperclip-bench-python" -version = "0.0.0" -description = "Placeholder bridge package for paperclip-bench Python evaluator integrations." -readme = "README.md" -requires-python = ">=3.11" -dependencies = [] - -[build-system] -requires = ["setuptools>=68"] -build-backend = "setuptools.build_meta" diff --git a/evals/bench/python/README.md b/evals/bench/python/README.md deleted file mode 100644 index faa453a0449..00000000000 --- a/evals/bench/python/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Python Bridge Layout - -This directory reserves the Python side of `paperclip-bench` for ecosystems and evaluator stacks that already expect Python entrypoints. - -- `bridge/`: future wrappers around the TypeScript runner and evaluator APIs -- `evaluators/`: Python-native evaluator adapters when a benchmark ecosystem already ships there -- `datasets/`: dataset preparation helpers that are easier to keep in Python diff --git a/evals/bench/python/bridge/.gitkeep b/evals/bench/python/bridge/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/python/bridge/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/python/datasets/.gitkeep b/evals/bench/python/datasets/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/python/datasets/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/python/evaluators/.gitkeep b/evals/bench/python/evaluators/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/python/evaluators/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/python/src/paperclip_bench/__init__.py b/evals/bench/python/src/paperclip_bench/__init__.py deleted file mode 100644 index a3489e2a6e4..00000000000 --- a/evals/bench/python/src/paperclip_bench/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Placeholder Python bridge package for paperclip-bench Phase 1.""" diff --git a/evals/bench/runs/.gitkeep b/evals/bench/runs/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/runs/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/runs/README.md b/evals/bench/runs/README.md deleted file mode 100644 index a2303d7e3c4..00000000000 --- a/evals/bench/runs/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Run Artifacts - -Generated benchmark runs belong in this directory. - -- Raw outputs remain untracked by default. -- Keep only placeholder files in git during Phase 1. diff --git a/evals/bench/scripts/validate-manifests.mjs b/evals/bench/scripts/validate-manifests.mjs deleted file mode 100644 index 356589fc7a5..00000000000 --- a/evals/bench/scripts/validate-manifests.mjs +++ /dev/null @@ -1,109 +0,0 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -import Ajv2020 from "ajv/dist/2020.js"; -import addFormats from "ajv-formats"; -import YAML from "yaml"; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const rootDir = path.resolve(__dirname, ".."); -const schemaPath = path.join(rootDir, "bench.schema.json"); - -async function main() { - const schema = JSON.parse(await fs.readFile(schemaPath, "utf8")); - const ajv = new Ajv2020({ allErrors: true, strict: false }); - addFormats(ajv); - - const schemaId = typeof schema.$id === "string" && schema.$id.length > 0 ? schema.$id : "paperclip-bench/bench.schema.json"; - ajv.addSchema(schema, schemaId); - - const validateTask = ajv.getSchema(`${schemaId}#/$defs/taskV1`) ?? ajv.compile({ $ref: `${schemaId}#/$defs/taskV1` }); - const validateSuite = ajv.getSchema(`${schemaId}#/$defs/suiteV1`) ?? ajv.compile({ $ref: `${schemaId}#/$defs/suiteV1` }); - - const manifestPaths = [ - ...(await collectYamlFiles(path.join(rootDir, "suites"))), - ...(await collectYamlFiles(path.join(rootDir, "benchmarks"))), - ]; - - if (manifestPaths.length === 0) { - throw new Error("No YAML manifests found to validate."); - } - - let failures = 0; - - for (const manifestPath of manifestPaths) { - const relativePath = path.relative(rootDir, manifestPath); - const doc = YAML.parse(await fs.readFile(manifestPath, "utf8")); - if (!doc || typeof doc !== "object") { - console.error(`FAIL ${relativePath}: expected YAML object`); - failures += 1; - continue; - } - - const schemaName = doc.schema; - if (schemaName === "paperclip-bench/task/v1") { - if (!validateTask(doc)) { - console.error(formatErrors(relativePath, validateTask.errors ?? [])); - failures += 1; - continue; - } - console.log(`OK ${relativePath} (task)`); - continue; - } - - if (schemaName === "paperclip-bench/suite/v1") { - if (!validateSuite(doc)) { - console.error(formatErrors(relativePath, validateSuite.errors ?? [])); - failures += 1; - continue; - } - console.log(`OK ${relativePath} (suite)`); - continue; - } - - if (schemaName == null) { - continue; - } - - console.error(`FAIL ${relativePath}\n - /schema: unsupported manifest schema ${JSON.stringify(schemaName)}`); - failures += 1; - } - - if (failures > 0) { - process.exitCode = 1; - return; - } - - console.log(`Validated ${manifestPaths.length} manifest(s).`); -} - -async function collectYamlFiles(dir) { - const entries = await fs.readdir(dir, { withFileTypes: true }); - const files = []; - for (const entry of entries) { - const entryPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - files.push(...(await collectYamlFiles(entryPath))); - continue; - } - if (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml")) { - files.push(entryPath); - } - } - return files; -} - -function formatErrors(relativePath, errors) { - const lines = [`FAIL ${relativePath}`]; - for (const error of errors) { - const instancePath = error.instancePath || "/"; - lines.push(` - ${instancePath}: ${error.message}`); - } - return lines.join("\n"); -} - -main().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -}); diff --git a/evals/bench/src/README.md b/evals/bench/src/README.md deleted file mode 100644 index 598496caebf..00000000000 --- a/evals/bench/src/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# TypeScript Runtime Layout - -These checked-in placeholder directories reserve stable homes for the future benchmark CLI and runtime code. - -- `cli/`: user-facing commands -- `core/`: manifest loading and run orchestration -- `datasets/`: imported benchmark normalization -- `runners/`: Paperclip, direct-agent, noop, and other execution backends -- `evaluators/`: command and benchmark-specific evaluator adapters -- `metrics/`: metric extraction and failure taxonomy -- `reports/`: aggregate report generation diff --git a/evals/bench/src/cli/.gitkeep b/evals/bench/src/cli/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/src/cli/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/src/core/.gitkeep b/evals/bench/src/core/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/src/core/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/src/datasets/.gitkeep b/evals/bench/src/datasets/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/src/datasets/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/src/evaluators/.gitkeep b/evals/bench/src/evaluators/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/src/evaluators/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/src/metrics/.gitkeep b/evals/bench/src/metrics/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/src/metrics/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/src/reports/.gitkeep b/evals/bench/src/reports/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/src/reports/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/src/runners/.gitkeep b/evals/bench/src/runners/.gitkeep deleted file mode 100644 index 8b137891791..00000000000 --- a/evals/bench/src/runners/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/bench/suites/paperclip-native-smoke.yaml b/evals/bench/suites/paperclip-native-smoke.yaml deleted file mode 100644 index 340c4aba67d..00000000000 --- a/evals/bench/suites/paperclip-native-smoke.yaml +++ /dev/null @@ -1,24 +0,0 @@ -$schema: ../bench.schema.json#/$defs/suiteV1 -schema: paperclip-bench/suite/v1 -id: paperclip-native-smoke -title: Paperclip-native smoke -description: Small native task suite for control-plane behavior, artifact generation, and starter-company wiring. -runner: - kind: paperclip - mode: isolated-instance -paperclip_version: current -model: anthropic/claude-sonnet-4.5 -tasks: - - ../benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml -defaults: - budget: - max_wall_clock_minutes: 120 - max_cost_usd: 10 - max_tool_calls: 300 - max_test_runs: 20 - stop: - stable_idle_minutes: 5 - require_terminal_issue_state: true - allow_human_intervention: false -notes: - - Uses the eng-squad-basic starter company package. diff --git a/evals/bench/suites/smoke.local.yaml b/evals/bench/suites/smoke.local.yaml deleted file mode 100644 index eedb236183d..00000000000 --- a/evals/bench/suites/smoke.local.yaml +++ /dev/null @@ -1,20 +0,0 @@ -$schema: ../bench.schema.json#/$defs/suiteV1 -schema: paperclip-bench/suite/v1 -id: smoke.local -title: Local scaffold smoke -description: Validate the manifest contract and exercise both imported and native task lanes with a placeholder runner. -runner: - kind: noop -paperclip_version: current -model: noop -tasks: - - ../benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml - - ../benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml -defaults: - budget: - max_wall_clock_minutes: 15 - max_cost_usd: 0 - stop: - stable_idle_minutes: 1 -notes: - - Phase 1 smoke suite for schema validation and scaffold review. diff --git a/evals/bench/suites/swebench-lite-smoke.yaml b/evals/bench/suites/swebench-lite-smoke.yaml deleted file mode 100644 index f411fade448..00000000000 --- a/evals/bench/suites/swebench-lite-smoke.yaml +++ /dev/null @@ -1,22 +0,0 @@ -$schema: ../bench.schema.json#/$defs/suiteV1 -schema: paperclip-bench/suite/v1 -id: swebench-lite-smoke -title: SWE-bench Lite smoke -description: Minimal imported-task suite that proves where SWE-bench Lite manifests belong and how they are referenced. -runner: - kind: prepare - mode: isolated-instance -paperclip_version: current -model: anthropic/claude-sonnet-4.5 -tasks: - - ../benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml -defaults: - budget: - max_wall_clock_minutes: 60 - max_cost_usd: 5 - max_tool_calls: 150 - stop: - stable_idle_minutes: 5 - require_terminal_issue_state: true -notes: - - Emits normalized metadata for imported tasks before the evaluator bridge lands. diff --git a/evals/bench/tasks/paperclip-native-smoke.jsonl b/evals/bench/tasks/paperclip-native-smoke.jsonl deleted file mode 100644 index 76e4dbf88e5..00000000000 --- a/evals/bench/tasks/paperclip-native-smoke.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"id":"paperclip-native-001","title":"Add request IDs to a checkout flow","problem_statement":"In a small Paperclip-like service, add request IDs to the issue checkout path so every checkout mutation can be correlated with logs and final review notes.","time_budget_minutes":60,"token_budget_usd":5,"success_checks":["Unit tests for request ID propagation pass","Final summary includes verification and remaining risks"],"human_rubric":{"architecture":5,"minimality":5,"test_quality":5,"final_summary":5},"required_artifacts":["implementation patch","test evidence","final PR summary","risk notes"]} diff --git a/package.json b/package.json index 312298281d2..d470705e7c3 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "test:e2e:headed": "npx playwright test --config tests/e2e/playwright.config.ts --headed", "test:e2e:multiuser-authenticated": "npx playwright test --config tests/e2e/playwright-multiuser-authenticated.config.ts", "evals:smoke": "cd evals/promptfoo && npx promptfoo@0.103.3 eval", - "bench:smoke": "pnpm paperclipai bench run --tasks evals/bench/tasks/paperclip-native-smoke.jsonl --benchmark paperclip_native_smoke --paperclip-version current --model noop --output .tmp/paperclip-bench-smoke --clean", "test:release-smoke": "npx playwright test --config tests/release-smoke/playwright.config.ts", "test:release-smoke:headed": "npx playwright test --config tests/release-smoke/playwright.config.ts --headed", "metrics:paperclip-commits": "tsx scripts/paperclip-commit-metrics.ts"