From 195091afac8276738f376991b01b6edc0219ce18 Mon Sep 17 00:00:00 2001
From: Dotta <bippadotta@protonmail.com>
Date: Fri, 24 Apr 2026 13:37:45 -0500
Subject: [PATCH] Polish benchmark scaffold layout

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 evals/bench/.gitignore                         |  1 +
 evals/bench/README.md                          |  8 ++++----
 evals/bench/ops/README.md                      |  7 +++++++
 evals/bench/ops/ci/.gitkeep                    |  1 +
 evals/bench/ops/docker/.gitkeep                |  1 +
 evals/bench/ops/single-machine-cloud/.gitkeep  |  1 +
 evals/bench/python/README.md                   |  7 +++++++
 evals/bench/python/bridge/.gitkeep             |  1 +
 evals/bench/python/datasets/.gitkeep           |  1 +
 evals/bench/python/evaluators/.gitkeep         |  1 +
 .../python/src/paperclip_bench/__init__.py     |  1 +
 evals/bench/runs/.gitkeep                      |  1 +
 evals/bench/runs/README.md                     |  6 ++++++
 evals/bench/scripts/validate-manifests.mjs     | 18 ++++++++++++++----
 evals/bench/src/README.md                      |  2 +-
 evals/bench/src/core/.gitkeep                  |  1 +
 evals/bench/src/datasets/.gitkeep              |  1 +
 evals/bench/src/evaluators/.gitkeep            |  1 +
 evals/bench/src/metrics/.gitkeep               |  1 +
 evals/bench/src/reports/.gitkeep               |  1 +
 evals/bench/src/runners/.gitkeep               |  1 +
 21 files changed, 54 insertions(+), 9 deletions(-)
 create mode 100644 evals/bench/ops/README.md
 create mode 100644 evals/bench/ops/ci/.gitkeep
 create mode 100644 evals/bench/ops/docker/.gitkeep
 create mode 100644 evals/bench/ops/single-machine-cloud/.gitkeep
 create mode 100644 evals/bench/python/README.md
 create mode 100644 evals/bench/python/bridge/.gitkeep
 create mode 100644 evals/bench/python/datasets/.gitkeep
 create mode 100644 evals/bench/python/evaluators/.gitkeep
 create mode 100644 evals/bench/python/src/paperclip_bench/__init__.py
 create mode 100644 evals/bench/runs/.gitkeep
 create mode 100644 evals/bench/runs/README.md
 create mode 100644 evals/bench/src/core/.gitkeep
 create mode 100644 evals/bench/src/datasets/.gitkeep
 create mode 100644 evals/bench/src/evaluators/.gitkeep
 create mode 100644 evals/bench/src/metrics/.gitkeep
 create mode 100644 evals/bench/src/reports/.gitkeep
 create mode 100644 evals/bench/src/runners/.gitkeep

diff --git a/evals/bench/.gitignore b/evals/bench/.gitignore
index d3189f1617a..23f7ddb4808 100644
--- a/evals/bench/.gitignore
+++ b/evals/bench/.gitignore
@@ -1,2 +1,3 @@
 runs/*
 !runs/.gitkeep
+!runs/README.md
diff --git a/evals/bench/README.md b/evals/bench/README.md
index 67330eb5c7d..579c4888c11 100644
--- a/evals/bench/README.md
+++ b/evals/bench/README.md
@@ -20,10 +20,10 @@ This phase does not implement the runner. It establishes:
 | `benchmarks/paperclip-native/` | Paperclip-native tasks, prompts, rubrics, evaluator config, and hidden-test placeholders. |
 | `companies/` | Starter company packages used to seed Paperclip-native benchmark runs. |
 | `fixtures/` | Fixture repo placeholders plus shared env/secrets examples. |
-| `src/` | Future TypeScript CLI, task normalization, runners, evaluators, metrics, and reports. |
-| `python/` | Future bridge scripts for ecosystems that already expect Python entrypoints. |
-| `ops/` | Local Docker and single-machine cloud execution scaffolding. |
-| `runs/` | Generated benchmark outputs. Raw runs stay untracked by default. |
+| `src/` | Future TypeScript CLI, task normalization, runners, evaluators, metrics, and reports. Placeholder subdirectories are checked in so contributors can see the intended module boundaries. |
+| `python/` | Future bridge scripts for ecosystems that already expect Python entrypoints. Placeholder subdirectories reserve Python bridge, evaluator, and dataset helpers. |
+| `ops/` | Local Docker and single-machine cloud execution scaffolding, plus future CI entrypoints. |
+| `runs/` | Generated benchmark outputs. Raw runs stay untracked by default, with only a placeholder kept in git. |
 
 ## Manifest Validation
 
diff --git a/evals/bench/ops/README.md b/evals/bench/ops/README.md
new file mode 100644
index 00000000000..99ae6127248
--- /dev/null
+++ b/evals/bench/ops/README.md
@@ -0,0 +1,7 @@
+# Ops Layout
+
+Operational scaffolding for benchmark execution lives here.
+
+- `docker/`: local container images and compose files for reproducible runs
+- `single-machine-cloud/`: scripts and templates for one-box hosted execution
+- `ci/`: future CI entrypoints and release plumbing for benchmark automation
diff --git a/evals/bench/ops/ci/.gitkeep b/evals/bench/ops/ci/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/ops/ci/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/ops/docker/.gitkeep b/evals/bench/ops/docker/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/ops/docker/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/ops/single-machine-cloud/.gitkeep b/evals/bench/ops/single-machine-cloud/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/ops/single-machine-cloud/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/python/README.md b/evals/bench/python/README.md
new file mode 100644
index 00000000000..faa453a0449
--- /dev/null
+++ b/evals/bench/python/README.md
@@ -0,0 +1,7 @@
+# Python Bridge Layout
+
+This directory reserves the Python side of `paperclip-bench` for ecosystems and evaluator stacks that already expect Python entrypoints.
+
+- `bridge/`: future wrappers around the TypeScript runner and evaluator APIs
+- `evaluators/`: Python-native evaluator adapters when a benchmark ecosystem already ships there
+- `datasets/`: dataset preparation helpers that are easier to keep in Python
diff --git a/evals/bench/python/bridge/.gitkeep b/evals/bench/python/bridge/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/python/bridge/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/python/datasets/.gitkeep b/evals/bench/python/datasets/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/python/datasets/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/python/evaluators/.gitkeep b/evals/bench/python/evaluators/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/python/evaluators/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/python/src/paperclip_bench/__init__.py b/evals/bench/python/src/paperclip_bench/__init__.py
new file mode 100644
index 00000000000..a3489e2a6e4
--- /dev/null
+++ b/evals/bench/python/src/paperclip_bench/__init__.py
@@ -0,0 +1 @@
+"""Placeholder Python bridge package for paperclip-bench Phase 1."""
diff --git a/evals/bench/runs/.gitkeep b/evals/bench/runs/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/runs/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/runs/README.md b/evals/bench/runs/README.md
new file mode 100644
index 00000000000..a2303d7e3c4
--- /dev/null
+++ b/evals/bench/runs/README.md
@@ -0,0 +1,6 @@
+# Run Artifacts
+
+Generated benchmark runs belong in this directory.
+
+- Raw outputs remain untracked by default.
+- Keep only placeholder files in git during Phase 1.
diff --git a/evals/bench/scripts/validate-manifests.mjs b/evals/bench/scripts/validate-manifests.mjs
index 4c9c99f3b5f..356589fc7a5 100644
--- a/evals/bench/scripts/validate-manifests.mjs
+++ b/evals/bench/scripts/validate-manifests.mjs
@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
-import Ajv from "ajv";
+import Ajv2020 from "ajv/dist/2020.js";
 import addFormats from "ajv-formats";
 import YAML from "yaml";
 
@@ -12,11 +12,14 @@ const schemaPath = path.join(rootDir, "bench.schema.json");
 
 async function main() {
   const schema = JSON.parse(await fs.readFile(schemaPath, "utf8"));
-  const ajv = new Ajv({ allErrors: true, strict: false });
+  const ajv = new Ajv2020({ allErrors: true, strict: false });
   addFormats(ajv);
 
-  const validateTask = ajv.compile(schema.$defs.taskV1);
-  const validateSuite = ajv.compile(schema.$defs.suiteV1);
+  const schemaId = typeof schema.$id === "string" && schema.$id.length > 0 ? schema.$id : "paperclip-bench/bench.schema.json";
+  ajv.addSchema(schema, schemaId);
+
+  const validateTask = ajv.getSchema(`${schemaId}#/$defs/taskV1`) ?? ajv.compile({ $ref: `${schemaId}#/$defs/taskV1` });
+  const validateSuite = ajv.getSchema(`${schemaId}#/$defs/suiteV1`) ?? ajv.compile({ $ref: `${schemaId}#/$defs/suiteV1` });
 
   const manifestPaths = [
     ...(await collectYamlFiles(path.join(rootDir, "suites"))),
@@ -58,6 +61,13 @@ async function main() {
       console.log(`OK   ${relativePath} (suite)`);
       continue;
     }
+
+    if (schemaName == null) {
+      continue;
+    }
+
+    console.error(`FAIL ${relativePath}\n  - /schema: unsupported manifest schema ${JSON.stringify(schemaName)}`);
+    failures += 1;
   }
 
   if (failures > 0) {
diff --git a/evals/bench/src/README.md b/evals/bench/src/README.md
index 4d8b99f002b..598496caebf 100644
--- a/evals/bench/src/README.md
+++ b/evals/bench/src/README.md
@@ -1,6 +1,6 @@
 # TypeScript Runtime Layout
 
-These directories reserve stable homes for the future benchmark CLI and runtime code.
+These checked-in placeholder directories reserve stable homes for the future benchmark CLI and runtime code.
 
 - `cli/`: user-facing commands
 - `core/`: manifest loading and run orchestration
diff --git a/evals/bench/src/core/.gitkeep b/evals/bench/src/core/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/src/core/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/src/datasets/.gitkeep b/evals/bench/src/datasets/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/src/datasets/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/src/evaluators/.gitkeep b/evals/bench/src/evaluators/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/src/evaluators/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/src/metrics/.gitkeep b/evals/bench/src/metrics/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/src/metrics/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/src/reports/.gitkeep b/evals/bench/src/reports/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/src/reports/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/evals/bench/src/runners/.gitkeep b/evals/bench/src/runners/.gitkeep
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/evals/bench/src/runners/.gitkeep
@@ -0,0 +1 @@
+