Remove superseded benchmark harness

cli/src/__tests__/bench.test.ts

@@ -1,345 +0,0 @@
-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import os from "node:os";
-import path from "node:path";
-import { describe, expect, it } from "vitest";
-import {
-  loadBenchTasks,
-  normalizeBenchRunner,
-  normalizeBenchTask,
-  renderBenchIssueMarkdown,
-  runPaperclipBench,
-} from "../commands/bench.js";
-
-describe("normalizeBenchTask", () => {
-  it("accepts MissionBench-style snake_case fields", () => {
-    const task = normalizeBenchTask(
-      {
-        id: "missionbench-001",
-        title: "Add audit logs",
-        repo: "github.com/paperclipai/missionbench-saas",
-        starting_commit: "abc123",
-        problem_statement: "Add organization-level audit logs.",
-        time_budget_minutes: 180,
-        token_budget_usd: 20,
-        success_checks: ["npm test", "npm run typecheck"],
-        human_rubric: { architecture: 5, test_quality: 5 },
-        required_artifacts: ["implementation patch", "risk notes"],
-      },
-      { lineLabel: "tasks.jsonl:1" },
-    );
-
-    expect(task).toMatchObject({
-      id: "missionbench-001",
-      title: "Add audit logs",
-      problemStatement: "Add organization-level audit logs.",
-      repo: {
-        url: "github.com/paperclipai/missionbench-saas",
-        baseCommit: "abc123",
-      },
-      maxWallClockMinutes: 180,
-      maxCostUsd: 20,
-      successChecks: ["npm test", "npm run typecheck"],
-      requiredArtifacts: ["implementation patch", "risk notes"],
-    });
-    expect(task.humanRubric).toEqual([
-      { name: "architecture", maxScore: 5 },
-      { name: "test_quality", maxScore: 5 },
-    ]);
-  });
-
-  it("rejects tasks without a problem statement", () => {
-    expect(() =>
-      normalizeBenchTask(
-        {
-          id: "missing-problem",
-          title: "Missing problem",
-        },
-        { lineLabel: "tasks.jsonl:2" },
-      ),
-    ).toThrow(/requires problemStatement/i);
-  });
-
-  it("keeps an explicit issue reference for reuse", () => {
-    const task = normalizeBenchTask(
-      {
-        id: "missionbench-002",
-        title: "Reuse benchmark issue",
-        problem_statement: "Continue work in an existing benchmark issue.",
-        paperclip_issue_id: "PAP-42",
-      },
-      { lineLabel: "tasks.jsonl:3" },
-    );
-
-    expect(task.paperclipIssueRef).toBe("PAP-42");
-  });
-});
-
-describe("normalizeBenchRunner", () => {
-  it("accepts prepare and paperclip", () => {
-    expect(normalizeBenchRunner(undefined)).toBe("prepare");
-    expect(normalizeBenchRunner("prepare")).toBe("prepare");
-    expect(normalizeBenchRunner("paperclip")).toBe("paperclip");
-  });
-
-  it("rejects unsupported runners", () => {
-    expect(() => normalizeBenchRunner("live")).toThrow(/Unsupported bench runner/i);
-  });
-});
-
-describe("renderBenchIssueMarkdown", () => {
-  it("renders a Paperclip-ready task prompt", () => {
-    const markdown = renderBenchIssueMarkdown(
-      {
-        id: "missionbench-001",
-        title: "Add audit logs",
-        problemStatement: "Add organization-level audit logs.",
-        repo: { url: "https://example.com/repo.git", baseCommit: "abc123" },
-        maxWallClockMinutes: 120,
-        maxCostUsd: 10,
-        successChecks: ["npm test"],
-        humanRubric: [{ name: "architecture", maxScore: 5 }],
-        requiredArtifacts: ["final PR summary"],
-      },
-      {
-        benchmark: "missionbench",
-        runId: "run-1",
-        model: "anthropic/claude-sonnet-4.5",
-        paperclipVersion: "current",
-      },
-    );
-
-    expect(markdown).toContain("# Add audit logs");
-    expect(markdown).toContain("- Task ID: `missionbench-001`");
-    expect(markdown).toContain("- URL: https://example.com/repo.git");
-    expect(markdown).toContain("- npm test");
-    expect(markdown).toContain("- architecture (5 pts)");
-  });
-});
-
-describe("runPaperclipBench", () => {
-  it("writes manifest, per-task prompts, traces, and results", async () => {
-    const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-test-"));
-    try {
-      const tasksPath = path.join(tmpDir, "tasks.jsonl");
-      const outputDir = path.join(tmpDir, "out");
-      await writeFile(
-        tasksPath,
-        `${JSON.stringify({
-          id: "task-1",
-          title: "Fix checkout",
-          problemStatement: "Make checkout idempotent.",
-          successChecks: ["pnpm test"],
-        })}\n`,
-        "utf8",
-      );
-
-      const manifest = await runPaperclipBench({
-        tasksPath,
-        benchmark: "paperclip_native",
-        outputDir,
-        runId: "test-run",
-        paperclipVersion: "current",
-        model: "noop",
-        runner: "prepare",
-        clean: true,
-      });
-
-      expect(manifest.taskCount).toBe(1);
-      expect(manifest.tasks[0]?.issuePath).toBe("tasks/task-1/issue.md");
-
-      const issueMarkdown = await readFile(path.join(outputDir, "tasks/task-1/issue.md"), "utf8");
-      expect(issueMarkdown).toContain("Make checkout idempotent.");
-
-      const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8");
-      expect(results).toContain('"status":"prepared"');
-
-      const reloadedTasks = await loadBenchTasks(tasksPath);
-      expect(reloadedTasks.map((task) => task.id)).toEqual(["task-1"]);
-    } finally {
-      await rm(tmpDir, { recursive: true, force: true });
-    }
-  });
-
-  it("records live Paperclip runner outputs", async () => {
-    const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-live-test-"));
-    try {
-      const tasksPath = path.join(tmpDir, "tasks.jsonl");
-      const outputDir = path.join(tmpDir, "out");
-      await writeFile(
-        tasksPath,
-        `${JSON.stringify({
-          id: "task-live-1",
-          title: "Ship audit logs",
-          problemStatement: "Add audit log support.",
-          successChecks: ["pnpm test"],
-        })}\n`,
-        "utf8",
-      );
-
-      const manifest = await runPaperclipBench({
-        tasksPath,
-        benchmark: "paperclip_native",
-        outputDir,
-        runId: "live-run",
-        paperclipVersion: "current",
-        model: "noop",
-        runner: "paperclip",
-        clean: true,
-        issueProvisioner: async () => ({
-          source: "created",
-          issue: {
-            id: "issue-1",
-            companyId: "company-1",
-            projectId: null,
-            projectWorkspaceId: null,
-            goalId: null,
-            parentId: null,
-            title: "Ship audit logs",
-            description: null,
-            status: "todo",
-            priority: "medium",
-            assigneeAgentId: "agent-1",
-            assigneeUserId: null,
-            checkoutRunId: null,
-            executionRunId: null,
-            executionAgentNameKey: null,
-            executionLockedAt: null,
-            createdByAgentId: null,
-            createdByUserId: null,
-            issueNumber: 1,
-            identifier: "PAP-1",
-            requestDepth: 0,
-            billingCode: null,
-            assigneeAdapterOverrides: null,
-            executionWorkspaceId: null,
-            executionWorkspacePreference: null,
-            executionWorkspaceSettings: null,
-            startedAt: null,
-            completedAt: null,
-            cancelledAt: null,
-            hiddenAt: null,
-            createdAt: new Date("2026-04-24T12:00:00Z"),
-            updatedAt: new Date("2026-04-24T12:00:00Z"),
-          },
-        }),
-        runLiveTask: async ({ taskDir, issue }) => {
-          await writeFile(path.join(taskDir, "patch.diff"), "diff --git a/file b/file\n", "utf8");
-          return {
-            issue: {
-              ...issue,
-              status: "done",
-              executionRunId: "run-2",
-              updatedAt: new Date("2026-04-24T12:02:00Z"),
-            },
-            runIds: ["run-1", "run-2"],
-            status: "done",
-            patchPath: "tasks/task-live-1/patch.diff",
-          };
-        },
-      });
-
-      expect(manifest.runner).toBe("paperclip");
-      expect(manifest.tasks[0]).toMatchObject({
-        status: "done",
-        patchPath: "tasks/task-live-1/patch.diff",
-        paperclipRunIds: ["run-1", "run-2"],
-        paperclipIssue: {
-          id: "issue-1",
-          identifier: "PAP-1",
-          status: "done",
-          source: "created",
-        },
-      });
-
-      const trace = await readFile(path.join(outputDir, "tasks/task-live-1/trace.json"), "utf8");
-      expect(trace).toContain('"paperclipRunIds": [');
-      expect(trace).toContain('"patchPath": "tasks/task-live-1/patch.diff"');
-
-      const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8");
-      expect(results).toContain('"status":"done"');
-      expect(results).toContain('"paperclipRunIds":["run-1","run-2"]');
-      expect(results).toContain('"patchPath":"tasks/task-live-1/patch.diff"');
-    } finally {
-      await rm(tmpDir, { recursive: true, force: true });
-    }
-  });
-
-  it("writes failed live-run results when the Paperclip runner errors", async () => {
-    const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bench-live-fail-test-"));
-    try {
-      const tasksPath = path.join(tmpDir, "tasks.jsonl");
-      const outputDir = path.join(tmpDir, "out");
-      await writeFile(
-        tasksPath,
-        `${JSON.stringify({
-          id: "task-live-2",
-          title: "Fail audit logs",
-          problemStatement: "Add audit log support.",
-          successChecks: ["pnpm test"],
-        })}\n`,
-        "utf8",
-      );
-
-      const manifest = await runPaperclipBench({
-        tasksPath,
-        benchmark: "paperclip_native",
-        outputDir,
-        runId: "live-fail-run",
-        paperclipVersion: "current",
-        model: "noop",
-        runner: "paperclip",
-        clean: true,
-        continueOnError: true,
-        issueProvisioner: async () => ({
-          source: "reused",
-          issue: {
-            id: "issue-2",
-            companyId: "company-1",
-            projectId: null,
-            projectWorkspaceId: null,
-            goalId: null,
-            parentId: null,
-            title: "Fail audit logs",
-            description: null,
-            status: "in_progress",
-            priority: "medium",
-            assigneeAgentId: "agent-1",
-            assigneeUserId: null,
-            checkoutRunId: null,
-            executionRunId: null,
-            executionAgentNameKey: null,
-            executionLockedAt: null,
-            createdByAgentId: null,
-            createdByUserId: null,
-            issueNumber: 2,
-            identifier: "PAP-2",
-            requestDepth: 0,
-            billingCode: null,
-            assigneeAdapterOverrides: null,
-            executionWorkspaceId: null,
-            executionWorkspacePreference: null,
-            executionWorkspaceSettings: null,
-            startedAt: null,
-            completedAt: null,
-            cancelledAt: null,
-            hiddenAt: null,
-            createdAt: new Date("2026-04-24T12:00:00Z"),
-            updatedAt: new Date("2026-04-24T12:00:00Z"),
-          },
-        }),
-        runLiveTask: async () => {
-          throw new Error("heartbeat invoke failed");
-        },
-      });
-
-      expect(manifest.tasks[0]?.status).toBe("failed");
-
-      const results = await readFile(path.join(outputDir, "results.jsonl"), "utf8");
-      expect(results).toContain('"status":"failed"');
-      expect(results).toContain('"failureStage":"paperclip_runner"');
-      expect(results).toContain('"notes":"heartbeat invoke failed"');
-    } finally {
-      await rm(tmpDir, { recursive: true, force: true });
-    }
-  });
-});

cli/src/index.ts

@@ -18,7 +18,6 @@ import { registerActivityCommands } from "./commands/client/activity.js";
 import { registerDashboardCommands } from "./commands/client/dashboard.js";
 import { registerRoutineCommands } from "./commands/routines.js";
 import { registerFeedbackCommands } from "./commands/client/feedback.js";
-import { registerBenchCommands } from "./commands/bench.js";
 import { applyDataDirOverride, type DataDirOptionLike } from "./config/data-dir.js";
 import { loadPaperclipEnvFile } from "./config/env.js";
 import { initTelemetryFromConfigFile, flushTelemetry } from "./telemetry.js";
@@ -148,7 +147,6 @@ registerActivityCommands(program);
 registerDashboardCommands(program);
 registerRoutineCommands(program);
 registerFeedbackCommands(program);
-registerBenchCommands(program);
 registerWorktreeCommands(program);
 registerEnvLabCommands(program);
 registerPluginCommands(program);

evals/README.md

@@ -34,53 +34,6 @@ promptfoo eval
 promptfoo view
 ```
 
-## Paperclip Bench
-
-`paperclipai bench` is the first-party benchmark harness for Paperclip-native and SWE-style task suites. It supports a deterministic `prepare` runner for prompt/render artifacts and a live `paperclip` runner that creates or reuses benchmark issues, wakes the assignee, and captures run metadata plus a final patch artifact.
-
-```bash
-# Prepare the bundled smoke suite
-pnpm bench:smoke
-
-# Validate any JSONL suite
-pnpm paperclipai bench validate --tasks evals/bench/tasks/paperclip-native-smoke.jsonl
-
-# Prepare artifacts for a custom suite
-pnpm paperclipai bench run \
-  --tasks path/to/tasks.jsonl \
-  --benchmark missionbench \
-  --paperclip-version current \
-  --model anthropic/claude-sonnet-4.5 \
-  --max-cost-usd 10 \
-  --max-wall-clock-minutes 120 \
-  --output .tmp/paperclip-bench/missionbench-run
-
-# Run tasks live against a Paperclip dev server
-pnpm paperclipai bench run \
-  --runner paperclip \
-  --tasks path/to/tasks.jsonl \
-  --benchmark missionbench \
-  --paperclip-version current \
-  --model anthropic/claude-sonnet-4.5 \
-  --company-id <company-id> \
-  --assignee-agent-id <agent-id> \
-  --project-id <project-id> \
-  --goal-id <goal-id> \
-  --max-cost-usd 10 \
-  --max-wall-clock-minutes 120 \
-  --output .tmp/paperclip-bench/missionbench-live
-```
-
-Task rows are JSONL objects. The parser accepts both camelCase and snake_case fields so suites can use MissionBench-style records:
-
-```json
-{"id":"missionbench-001","title":"Add audit logs","repo":"github.com/paperclipai/missionbench-saas","starting_commit":"abc123","problem_statement":"Add organization-level audit logs.","success_checks":["npm test"],"required_artifacts":["implementation patch","final PR summary"]}
-```
-
-Set `paperclip_issue_id`, `paperclip_issue_identifier`, `paperclipIssueRef`, or the same keys inside `metadata` to reuse an existing benchmark issue from the task row. Otherwise, `--create-issues -C <company-id>` creates one Paperclip issue per task for the `prepare` runner, and the live `paperclip` runner creates issues automatically when the task row does not already reference one. Created issues use the normalized prompt as the issue description and set `billingCode` to `bench:<run-id>` unless `--billing-code` is provided.
-
-Live-run `results.jsonl` and per-task `trace.json` include the Paperclip issue id, observed heartbeat run ids, the final task status, a relative `patchPath` when a git diff was extracted from the execution workspace, and a `failureStage` when the task stops short of a clean `done` result.
-
 ### What's tested
 
 Phase 0 covers narrow behavior evals for the Paperclip heartbeat skill:

evals/bench/.gitignore

@@ -1,3 +0,0 @@
-runs/*
-!runs/.gitkeep
-!runs/README.md

evals/bench/README.md

@@ -1,50 +0,0 @@
-# paperclip-bench
-
-`paperclip-bench` is the scaffold for Paperclip's benchmark harness and benchmark content. Phase 1 keeps it inside the monorepo under `evals/bench/`, but the layout is intentionally shaped so it can move into a standalone repository later with minimal churn.
-
-This phase does not implement the runner. It establishes:
-
-- where imported benchmark manifests belong
-- where Paperclip-native tasks belong
-- where starter company packages belong
-- where runner, evaluator, Python bridge, ops, and generated run artifacts belong
-- the initial schema contract for task and suite manifests
-
-## Layout
-
-| Path | Purpose |
-|---|---|
-| `bench.schema.json` | Shared JSON Schema definitions for `paperclip-bench/task/v1` and `paperclip-bench/suite/v1`. |
-| `suites/` | Versioned suite manifests such as local smoke, SWE-bench Lite smoke, and Paperclip-native smoke. |
-| `benchmarks/imported/` | Imported benchmark metadata and normalized task manifests without vendoring third-party source repositories. |
-| `benchmarks/paperclip-native/` | Paperclip-native tasks, prompts, rubrics, evaluator config, and hidden-test placeholders. |
-| `companies/` | Starter company packages used to seed Paperclip-native benchmark runs. |
-| `fixtures/` | Fixture repo placeholders plus shared env/secrets examples. |
-| `src/` | Future TypeScript CLI, task normalization, runners, evaluators, metrics, and reports. Placeholder subdirectories are checked in so contributors can see the intended module boundaries. |
-| `python/` | Future bridge scripts for ecosystems that already expect Python entrypoints. Placeholder subdirectories reserve Python bridge, evaluator, and dataset helpers. |
-| `ops/` | Local Docker and single-machine cloud execution scaffolding, plus future CI entrypoints. |
-| `runs/` | Generated benchmark outputs. Raw runs stay untracked by default, with only a placeholder kept in git. |
-
-## Manifest Validation
-
-Phase 1 picks JSON Schema plus YAML manifests. The validation script loads YAML, validates against `bench.schema.json`, and checks both suite and task examples:
-
-```bash
-node evals/bench/scripts/validate-manifests.mjs
-# or
-pnpm --dir evals/bench run validate:manifests
-```
-
-## V0 Verification Intent
-
-The first vertical slice will be considered wired correctly when these commands are all true:
-
-1. `node evals/bench/scripts/validate-manifests.mjs` passes.
-2. A contributor can locate imported tasks, native tasks, starter companies, runner code, Python bridge code, ops files, and generated runs from this directory layout alone.
-3. Later phases can add implementation in place without changing manifest ids or top-level directory meaning.
-
-## Notes
-
-- `.paperclip.yaml` starter-company files are placeholders in this phase. Their exact import/runtime contract lands with the Paperclip runner integration work.
-- Hidden tests are represented as directories only. Real evaluator wiring lands in later phases.
-- The current Paperclip CLI already has early bench commands under `cli/src/commands/bench.ts`; this scaffold defines the repository contract those commands can grow into.

evals/bench/bench.schema.json

@@ -1,272 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://paperclip.ai/schemas/paperclip-bench/bench.schema.json",
-  "title": "paperclip-bench manifest schema",
-  "$defs": {
-    "pathRef": {
-      "type": "string",
-      "minLength": 1
-    },
-    "runnerKind": {
-      "type": "string",
-      "enum": ["paperclip", "direct-agent", "noop", "prepare"]
-    },
-    "repo": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["kind", "url", "ref"],
-      "properties": {
-        "kind": {
-          "type": "string",
-          "enum": ["git", "local-fixture"]
-        },
-        "url": {
-          "type": "string",
-          "minLength": 1
-        },
-        "ref": {
-          "type": "string",
-          "minLength": 1
-        },
-        "workdir": {
-          "type": "string",
-          "minLength": 1
-        }
-      }
-    },
-    "budget": {
-      "type": "object",
-      "additionalProperties": false,
-      "properties": {
-        "max_wall_clock_minutes": {
-          "type": "integer",
-          "minimum": 1
-        },
-        "max_cost_usd": {
-          "type": "number",
-          "minimum": 0
-        },
-        "max_tool_calls": {
-          "type": "integer",
-          "minimum": 1
-        },
-        "max_test_runs": {
-          "type": "integer",
-          "minimum": 1
-        }
-      }
-    },
-    "stop": {
-      "type": "object",
-      "additionalProperties": false,
-      "properties": {
-        "stable_idle_minutes": {
-          "type": "integer",
-          "minimum": 0
-        },
-        "require_terminal_issue_state": {
-          "type": "boolean"
-        },
-        "allow_human_intervention": {
-          "type": "boolean"
-        }
-      }
-    },
-    "runner": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["kind"],
-      "properties": {
-        "kind": {
-          "$ref": "#/$defs/runnerKind"
-        },
-        "mode": {
-          "type": "string",
-          "minLength": 1
-        }
-      }
-    },
-    "evaluator": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["kind"],
-      "properties": {
-        "kind": {
-          "type": "string",
-          "enum": ["command", "swebench", "swebench-pro", "paperclip-native"]
-        },
-        "manifest": {
-          "$ref": "#/$defs/pathRef"
-        },
-        "prediction_format": {
-          "type": "string",
-          "minLength": 1
-        },
-        "visible_commands": {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minLength": 1
-          }
-        },
-        "hidden_commands": {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minLength": 1
-          }
-        }
-      }
-    },
-    "taskV1": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["schema", "id", "family", "summary", "repo", "runner", "evaluator"],
-      "properties": {
-        "$schema": {
-          "type": "string",
-          "minLength": 1
-        },
-        "schema": {
-          "type": "string",
-          "const": "paperclip-bench/task/v1"
-        },
-        "id": {
-          "type": "string",
-          "pattern": "^[a-z0-9][a-z0-9._:-]*$"
-        },
-        "family": {
-          "type": "string",
-          "minLength": 1
-        },
-        "summary": {
-          "type": "string",
-          "minLength": 1
-        },
-        "description": {
-          "type": "string"
-        },
-        "source": {
-          "type": "object"
-        },
-        "repo": {
-          "$ref": "#/$defs/repo"
-        },
-        "prompt": {
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "path": {
-              "$ref": "#/$defs/pathRef"
-            }
-          }
-        },
-        "starter_company": {
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "package": {
-              "$ref": "#/$defs/pathRef"
-            }
-          }
-        },
-        "runner": {
-          "$ref": "#/$defs/runner"
-        },
-        "budget": {
-          "$ref": "#/$defs/budget"
-        },
-        "stop": {
-          "$ref": "#/$defs/stop"
-        },
-        "evaluator": {
-          "$ref": "#/$defs/evaluator"
-        },
-        "required_artifacts": {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minLength": 1
-          }
-        },
-        "metrics_profile": {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minLength": 1
-          }
-        },
-        "tags": {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minLength": 1
-          }
-        }
-      }
-    },
-    "suiteV1": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["schema", "id", "title", "tasks"],
-      "properties": {
-        "$schema": {
-          "type": "string",
-          "minLength": 1
-        },
-        "schema": {
-          "type": "string",
-          "const": "paperclip-bench/suite/v1"
-        },
-        "id": {
-          "type": "string",
-          "pattern": "^[a-z0-9][a-z0-9._-]*$"
-        },
-        "title": {
-          "type": "string",
-          "minLength": 1
-        },
-        "description": {
-          "type": "string"
-        },
-        "runner": {
-          "$ref": "#/$defs/runner"
-        },
-        "paperclip_version": {
-          "type": "string",
-          "minLength": 1
-        },
-        "model": {
-          "type": "string",
-          "minLength": 1
-        },
-        "tasks": {
-          "type": "array",
-          "minItems": 1,
-          "items": {
-            "$ref": "#/$defs/pathRef"
-          }
-        },
-        "defaults": {
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "budget": {
-              "$ref": "#/$defs/budget"
-            },
-            "stop": {
-              "$ref": "#/$defs/stop"
-            }
-          }
-        },
-        "notes": {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "minLength": 1
-          }
-        }
-      }
-    }
-  }
-}

evals/bench/benchmarks/imported/swebench-lite/README.md

@@ -1,5 +0,0 @@
-# SWE-bench Lite Imports
-
-This directory stores normalized task manifests and dataset notes for imported SWE-bench Lite tasks. It does not vendor upstream benchmark repositories or evaluator code.
-
-Use `tasks/` for one manifest per selected instance. Keep raw upstream metadata under `source` so later bridge code can emit official prediction formats without guessing.

evals/bench/benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml

@@ -1,39 +0,0 @@
-$schema: ../../../../bench.schema.json#/$defs/taskV1
-schema: paperclip-bench/task/v1
-id: swebench-lite:pytest-dev__pytest-6116
-family: swebench-lite
-summary: Fix a small pytest CLI behavior regression imported from SWE-bench Lite.
-description: Imported smoke task used to validate the scaffold and preserve upstream task metadata.
-source:
-  dataset: SWE-bench/SWE-bench_Lite
-  instance_id: pytest-dev__pytest-6116
-  repo: pytest-dev/pytest
-  base_commit: 17f7b44f2f1b8b1a6b2cb67f7e6f01a8a5b9ea12
-  version: "6.0"
-repo:
-  kind: git
-  url: https://github.com/pytest-dev/pytest.git
-  ref: 17f7b44f2f1b8b1a6b2cb67f7e6f01a8a5b9ea12
-runner:
-  kind: prepare
-  mode: isolated-instance
-budget:
-  max_wall_clock_minutes: 60
-  max_cost_usd: 5
-  max_tool_calls: 150
-stop:
-  stable_idle_minutes: 5
-  require_terminal_issue_state: true
-evaluator:
-  kind: swebench
-  prediction_format: swebench-jsonl
-required_artifacts:
-  - prediction.patch
-  - results.json
-metrics_profile:
-  - patch
-  - cost
-  - evaluator
-tags:
-  - imported
-  - smoke

evals/bench/benchmarks/paperclip-native/README.md

@@ -1,3 +0,0 @@
-# Paperclip-Native Tasks
-
-Paperclip-native tasks test the control plane itself: decomposition, blockers, handoffs, artifact quality, liveness, and reviewability. Each task owns its prompt, rubric, evaluator description, and hidden-test placeholders in one directory.

evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/evaluator.yaml

@@ -1,8 +0,0 @@
-kind: command
-visible_commands:
-  - pnpm test
-  - pnpm -r typecheck
-hidden_commands:
-  - pnpm test:e2e:audit-log
-notes:
-  - Hidden tests stay outside the agent workspace once evaluator wiring lands.

evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/hidden-tests/README.md

@@ -1,3 +0,0 @@
-# Hidden Tests Placeholder
-
-Real hidden tests are intentionally omitted in Phase 1. This directory exists so later phases have a stable location for evaluator-only assets.

evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/issue.md

@@ -1,10 +0,0 @@
-# Add Organization Audit Logs
-
-Implement organization-level audit logs for account settings changes in the demo SaaS fixture.
-
-Definition of done:
-
-- account settings mutations create durable audit-log records
-- the app exposes enough UI or API surface for operators to inspect the new entries
-- relevant tests are updated or added
-- the final summary names what changed, what was verified, and remaining risks

evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/rubric.md

@@ -1,9 +0,0 @@
-# Rubric
-
-Score from 0-5 on:
-
-- architecture
-- minimality
-- test quality
-- final summary quality
-- operational reviewability

evals/bench/benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml

@@ -1,49 +0,0 @@
-$schema: ../../../../bench.schema.json#/$defs/taskV1
-schema: paperclip-bench/task/v1
-id: pclip-native-001-audit-log-mission
-family: paperclip-native
-summary: Add organization-level audit logs to a demo SaaS app through a small engineering squad.
-description: Native smoke task focused on implementation, verification, and final review artifacts.
-repo:
-  kind: local-fixture
-  url: ../../../../fixtures/repos/demo-saas
-  ref: scaffold
-  workdir: .
-prompt:
-  path: ./issue.md
-starter_company:
-  package: ../../../../companies/eng-squad-basic
-runner:
-  kind: paperclip
-  mode: isolated-instance
-budget:
-  max_wall_clock_minutes: 120
-  max_cost_usd: 10
-  max_tool_calls: 300
-  max_test_runs: 20
-stop:
-  stable_idle_minutes: 5
-  require_terminal_issue_state: true
-  allow_human_intervention: false
-evaluator:
-  kind: command
-  manifest: ./evaluator.yaml
-  visible_commands:
-    - pnpm test
-    - pnpm -r typecheck
-  hidden_commands:
-    - pnpm test:e2e:audit-log
-required_artifacts:
-  - prediction.patch
-  - final_summary.md
-  - tests_run.md
-  - risks.md
-metrics_profile:
-  - patch
-  - cost
-  - paperclip-control-plane
-  - liveness
-  - reviewability
-tags:
-  - native
-  - smoke

evals/bench/companies/README.md

@@ -1,3 +0,0 @@
-# Starter Companies
-
-Starter company packages seed Paperclip-native tasks with a known initial org state. These directories follow the Agent Companies markdown package shape and add a provisional `.paperclip.yaml` file for benchmark-specific hints.

evals/bench/companies/ceo-only/.paperclip.yaml

@@ -1,5 +0,0 @@
-version: 1
-company_name: CEO Only
-benchmark:
-  enable_agent_timers: false
-  intended_runner: paperclip

evals/bench/companies/ceo-only/COMPANY.md

@@ -1,7 +0,0 @@
-# CEO Only
-
-Purpose: minimal bootstrap company for imported or direct-agent style tasks.
-
-Roles:
-
-- CEO owns the single benchmark issue.

evals/bench/companies/ceo-only/agents/ceo/AGENTS.md

@@ -1,9 +0,0 @@
-# CEO
-
-You own the benchmark issue directly.
-
-Default expectations:
-
-- read the task prompt
-- do the work or delegate when the harness allows it
-- finish with reviewable artifacts

evals/bench/companies/eng-squad-basic/.paperclip.yaml

@@ -1,6 +0,0 @@
-version: 1
-company_name: Engineering Squad Basic
-benchmark:
-  enable_agent_timers: false
-  intended_runner: paperclip
-  default_idle_stability_minutes: 5

evals/bench/companies/eng-squad-basic/COMPANY.md

@@ -1,10 +0,0 @@
-# Engineering Squad Basic
-
-Purpose: small delivery-focused company package for Paperclip-native smoke tasks.
-
-Roles:
-
-- CEO for coordination
-- CTO for task routing and technical review
-- Senior Engineer for implementation
-- QA for verification

evals/bench/companies/eng-squad-basic/agents/ceo/AGENTS.md

@@ -1,3 +0,0 @@
-# CEO
-
-Coordinate the benchmark mission, keep work moving, and ensure the final artifact set is reviewable.

evals/bench/companies/eng-squad-basic/agents/cto/AGENTS.md

@@ -1,3 +0,0 @@
-# CTO
-
-Break the mission into concrete engineering work, keep technical reviews crisp, and avoid unnecessary task churn.

evals/bench/companies/eng-squad-basic/agents/qa/AGENTS.md

@@ -1,3 +0,0 @@
-# QA
-
-Validate the deliverable against visible checks, inspect the final summary, and report residual risks.

evals/bench/companies/eng-squad-basic/agents/senior-engineer/AGENTS.md

@@ -1,3 +0,0 @@
-# Senior Engineer
-
-Implement the requested change, run relevant validation, and leave clear handoff notes for review.

evals/bench/companies/governed-eng-squad/.paperclip.yaml

@@ -1,6 +0,0 @@
-version: 1
-company_name: Governed Engineering Squad
-benchmark:
-  enable_agent_timers: false
-  intended_runner: paperclip
-  requires_review_handoffs: true

evals/bench/companies/governed-eng-squad/COMPANY.md

@@ -1,12 +0,0 @@
-# Governed Engineering Squad
-
-Purpose: review-heavy starter package for blocker and handoff benchmarks.
-
-Roles:
-
-- CEO
-- CTO
-- Senior Engineer
-- Security Engineer
-- UX Designer
-- QA

evals/bench/companies/governed-eng-squad/agents/ceo/AGENTS.md

@@ -1,3 +0,0 @@
-# CEO
-
-Own the benchmark outcome, keep review dependencies explicit, and avoid hidden state.

evals/bench/companies/governed-eng-squad/agents/cto/AGENTS.md

@@ -1,3 +0,0 @@
-# CTO
-
-Manage implementation sequencing, review dependencies, and blocker hygiene.

evals/bench/companies/governed-eng-squad/agents/qa/AGENTS.md

@@ -1,3 +0,0 @@
-# QA
-
-Validate acceptance criteria, inspect artifacts, and report whether the mission is ready for review.

evals/bench/companies/governed-eng-squad/agents/security-engineer/AGENTS.md

@@ -1,3 +0,0 @@
-# Security Engineer
-
-Review security-sensitive changes, document required controls, and block work explicitly when requirements are incomplete.

evals/bench/companies/governed-eng-squad/agents/senior-engineer/AGENTS.md

@@ -1,3 +0,0 @@
-# Senior Engineer
-
-Implement the technical work after required reviews are in place and leave evidence for QA.

evals/bench/companies/governed-eng-squad/agents/ux-designer/AGENTS.md

@@ -1,3 +0,0 @@
-# UX Designer
-
-Review operator-facing states and copy so benchmark tasks can measure design-review handoffs.

evals/bench/fixtures/README.md

@@ -1,3 +0,0 @@
-# Fixtures
-
-Fixture repos and shared env examples for benchmark tasks live here. Real fixture repositories land in later phases; Phase 1 keeps stable placeholders so tasks can reference durable paths.

evals/bench/fixtures/repos/demo-saas/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/fixtures/repos/workflow-demo/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/fixtures/secrets.example.env

@@ -1,5 +0,0 @@
-# Example env file for local benchmark runs.
-# Real values should be injected per run, not committed.
-OPENAI_API_KEY=
-ANTHROPIC_API_KEY=
-PAPERCLIP_API_URL=http://localhost:3100

evals/bench/ops/README.md

@@ -1,7 +0,0 @@
-# Ops Layout
-
-Operational scaffolding for benchmark execution lives here.
-
-- `docker/`: local container images and compose files for reproducible runs
-- `single-machine-cloud/`: scripts and templates for one-box hosted execution
-- `ci/`: future CI entrypoints and release plumbing for benchmark automation

evals/bench/ops/ci/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/ops/docker/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/ops/single-machine-cloud/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/package.json

@@ -1,16 +0,0 @@
-{
-  "name": "paperclip-bench",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "description": "Scaffold for Paperclip benchmark suites, tasks, and runner assets.",
-  "scripts": {
-    "validate:manifests": "node ./scripts/validate-manifests.mjs"
-  },
-  "devDependencies": {
-    "ajv": "^8.17.1",
-    "ajv-formats": "^3.0.1",
-    "yaml": "^2.8.1"
-  },
-  "packageManager": "pnpm@9.15.4"
-}

evals/bench/pyproject.toml

@@ -1,11 +0,0 @@
-[project]
-name = "paperclip-bench-python"
-version = "0.0.0"
-description = "Placeholder bridge package for paperclip-bench Python evaluator integrations."
-readme = "README.md"
-requires-python = ">=3.11"
-dependencies = []
-
-[build-system]
-requires = ["setuptools>=68"]
-build-backend = "setuptools.build_meta"

evals/bench/python/README.md

@@ -1,7 +0,0 @@
-# Python Bridge Layout
-
-This directory reserves the Python side of `paperclip-bench` for ecosystems and evaluator stacks that already expect Python entrypoints.
-
-- `bridge/`: future wrappers around the TypeScript runner and evaluator APIs
-- `evaluators/`: Python-native evaluator adapters when a benchmark ecosystem already ships there
-- `datasets/`: dataset preparation helpers that are easier to keep in Python

evals/bench/python/bridge/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/python/datasets/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/python/evaluators/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/python/src/paperclip_bench/__init__.py

@@ -1 +0,0 @@
-"""Placeholder Python bridge package for paperclip-bench Phase 1."""

evals/bench/runs/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/runs/README.md

@@ -1,6 +0,0 @@
-# Run Artifacts
-
-Generated benchmark runs belong in this directory.
-
-- Raw outputs remain untracked by default.
-- Keep only placeholder files in git during Phase 1.

evals/bench/scripts/validate-manifests.mjs

@@ -1,109 +0,0 @@
-import fs from "node:fs/promises";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-import Ajv2020 from "ajv/dist/2020.js";
-import addFormats from "ajv-formats";
-import YAML from "yaml";
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const rootDir = path.resolve(__dirname, "..");
-const schemaPath = path.join(rootDir, "bench.schema.json");
-
-async function main() {
-  const schema = JSON.parse(await fs.readFile(schemaPath, "utf8"));
-  const ajv = new Ajv2020({ allErrors: true, strict: false });
-  addFormats(ajv);
-
-  const schemaId = typeof schema.$id === "string" && schema.$id.length > 0 ? schema.$id : "paperclip-bench/bench.schema.json";
-  ajv.addSchema(schema, schemaId);
-
-  const validateTask = ajv.getSchema(`${schemaId}#/$defs/taskV1`) ?? ajv.compile({ $ref: `${schemaId}#/$defs/taskV1` });
-  const validateSuite = ajv.getSchema(`${schemaId}#/$defs/suiteV1`) ?? ajv.compile({ $ref: `${schemaId}#/$defs/suiteV1` });
-
-  const manifestPaths = [
-    ...(await collectYamlFiles(path.join(rootDir, "suites"))),
-    ...(await collectYamlFiles(path.join(rootDir, "benchmarks"))),
-  ];
-
-  if (manifestPaths.length === 0) {
-    throw new Error("No YAML manifests found to validate.");
-  }
-
-  let failures = 0;
-
-  for (const manifestPath of manifestPaths) {
-    const relativePath = path.relative(rootDir, manifestPath);
-    const doc = YAML.parse(await fs.readFile(manifestPath, "utf8"));
-    if (!doc || typeof doc !== "object") {
-      console.error(`FAIL ${relativePath}: expected YAML object`);
-      failures += 1;
-      continue;
-    }
-
-    const schemaName = doc.schema;
-    if (schemaName === "paperclip-bench/task/v1") {
-      if (!validateTask(doc)) {
-        console.error(formatErrors(relativePath, validateTask.errors ?? []));
-        failures += 1;
-        continue;
-      }
-      console.log(`OK   ${relativePath} (task)`);
-      continue;
-    }
-
-    if (schemaName === "paperclip-bench/suite/v1") {
-      if (!validateSuite(doc)) {
-        console.error(formatErrors(relativePath, validateSuite.errors ?? []));
-        failures += 1;
-        continue;
-      }
-      console.log(`OK   ${relativePath} (suite)`);
-      continue;
-    }
-
-    if (schemaName == null) {
-      continue;
-    }
-
-    console.error(`FAIL ${relativePath}\n  - /schema: unsupported manifest schema ${JSON.stringify(schemaName)}`);
-    failures += 1;
-  }
-
-  if (failures > 0) {
-    process.exitCode = 1;
-    return;
-  }
-
-  console.log(`Validated ${manifestPaths.length} manifest(s).`);
-}
-
-async function collectYamlFiles(dir) {
-  const entries = await fs.readdir(dir, { withFileTypes: true });
-  const files = [];
-  for (const entry of entries) {
-    const entryPath = path.join(dir, entry.name);
-    if (entry.isDirectory()) {
-      files.push(...(await collectYamlFiles(entryPath)));
-      continue;
-    }
-    if (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml")) {
-      files.push(entryPath);
-    }
-  }
-  return files;
-}
-
-function formatErrors(relativePath, errors) {
-  const lines = [`FAIL ${relativePath}`];
-  for (const error of errors) {
-    const instancePath = error.instancePath || "/";
-    lines.push(`  - ${instancePath}: ${error.message}`);
-  }
-  return lines.join("\n");
-}
-
-main().catch((error) => {
-  console.error(error instanceof Error ? error.message : String(error));
-  process.exitCode = 1;
-});

evals/bench/src/README.md

@@ -1,11 +0,0 @@
-# TypeScript Runtime Layout
-
-These checked-in placeholder directories reserve stable homes for the future benchmark CLI and runtime code.
-
-- `cli/`: user-facing commands
-- `core/`: manifest loading and run orchestration
-- `datasets/`: imported benchmark normalization
-- `runners/`: Paperclip, direct-agent, noop, and other execution backends
-- `evaluators/`: command and benchmark-specific evaluator adapters
-- `metrics/`: metric extraction and failure taxonomy
-- `reports/`: aggregate report generation

evals/bench/src/cli/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/src/core/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/src/datasets/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/src/evaluators/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/src/metrics/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/src/reports/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/src/runners/.gitkeep

@@ -1 +0,0 @@
-

evals/bench/suites/paperclip-native-smoke.yaml

@@ -1,24 +0,0 @@
-$schema: ../bench.schema.json#/$defs/suiteV1
-schema: paperclip-bench/suite/v1
-id: paperclip-native-smoke
-title: Paperclip-native smoke
-description: Small native task suite for control-plane behavior, artifact generation, and starter-company wiring.
-runner:
-  kind: paperclip
-  mode: isolated-instance
-paperclip_version: current
-model: anthropic/claude-sonnet-4.5
-tasks:
-  - ../benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml
-defaults:
-  budget:
-    max_wall_clock_minutes: 120
-    max_cost_usd: 10
-    max_tool_calls: 300
-    max_test_runs: 20
-  stop:
-    stable_idle_minutes: 5
-    require_terminal_issue_state: true
-    allow_human_intervention: false
-notes:
-  - Uses the eng-squad-basic starter company package.

evals/bench/suites/smoke.local.yaml

@@ -1,20 +0,0 @@
-$schema: ../bench.schema.json#/$defs/suiteV1
-schema: paperclip-bench/suite/v1
-id: smoke.local
-title: Local scaffold smoke
-description: Validate the manifest contract and exercise both imported and native task lanes with a placeholder runner.
-runner:
-  kind: noop
-paperclip_version: current
-model: noop
-tasks:
-  - ../benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml
-  - ../benchmarks/paperclip-native/tasks/pclip-native-001-audit-log-mission/task.yaml
-defaults:
-  budget:
-    max_wall_clock_minutes: 15
-    max_cost_usd: 0
-  stop:
-    stable_idle_minutes: 1
-notes:
-  - Phase 1 smoke suite for schema validation and scaffold review.

evals/bench/suites/swebench-lite-smoke.yaml

@@ -1,22 +0,0 @@
-$schema: ../bench.schema.json#/$defs/suiteV1
-schema: paperclip-bench/suite/v1
-id: swebench-lite-smoke
-title: SWE-bench Lite smoke
-description: Minimal imported-task suite that proves where SWE-bench Lite manifests belong and how they are referenced.
-runner:
-  kind: prepare
-  mode: isolated-instance
-paperclip_version: current
-model: anthropic/claude-sonnet-4.5
-tasks:
-  - ../benchmarks/imported/swebench-lite/tasks/pytest-dev__pytest-6116.yaml
-defaults:
-  budget:
-    max_wall_clock_minutes: 60
-    max_cost_usd: 5
-    max_tool_calls: 150
-  stop:
-    stable_idle_minutes: 5
-    require_terminal_issue_state: true
-notes:
-  - Emits normalized metadata for imported tasks before the evaluator bridge lands.

evals/bench/tasks/paperclip-native-smoke.jsonl

@@ -1 +0,0 @@
-{"id":"paperclip-native-001","title":"Add request IDs to a checkout flow","problem_statement":"In a small Paperclip-like service, add request IDs to the issue checkout path so every checkout mutation can be correlated with logs and final review notes.","time_budget_minutes":60,"token_budget_usd":5,"success_checks":["Unit tests for request ID propagation pass","Final summary includes verification and remaining risks"],"human_rubric":{"architecture":5,"minimality":5,"test_quality":5,"final_summary":5},"required_artifacts":["implementation patch","test evidence","final PR summary","risk notes"]}

package.json

@@ -39,7 +39,6 @@
     "test:e2e:headed": "npx playwright test --config tests/e2e/playwright.config.ts --headed",
     "test:e2e:multiuser-authenticated": "npx playwright test --config tests/e2e/playwright-multiuser-authenticated.config.ts",
     "evals:smoke": "cd evals/promptfoo && npx promptfoo@0.103.3 eval",
-    "bench:smoke": "pnpm paperclipai bench run --tasks evals/bench/tasks/paperclip-native-smoke.jsonl --benchmark paperclip_native_smoke --paperclip-version current --model noop --output .tmp/paperclip-bench-smoke --clean",
     "test:release-smoke": "npx playwright test --config tests/release-smoke/playwright.config.ts",
     "test:release-smoke:headed": "npx playwright test --config tests/release-smoke/playwright.config.ts --headed",
     "metrics:paperclip-commits": "tsx scripts/paperclip-commit-metrics.ts"