mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 19:35:13 +01:00
Dotta
5458940a6e
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Paperclip Runner needs repeatable evaluation contracts. > - Evaluation code must stay separate from provider launch and production orchestration. > - Offline fixtures need stable compatibility, scoring, traceability, and report rules. > - Published Runner consumers need only the supported evaluation contract surface. > - This pull request adds offline evaluation tooling and a workspace-private matrix kernel. > - The benefit is deterministic evaluation without credentials or paid provider calls. ## Linked Issues or Issue Description Refs #11297 This pull request extracts the offline evaluation unit from the earlier aggregate Runner work. ## What Changed - Add a workspace-private, provider-neutral evaluation matrix kernel. - Add the public `@paperclipai/paperclip-runner/evals` compatibility and native execution contracts. - Add fail-closed runnerd artifact and protocol compatibility checks. - Add deterministic workflow catalogs, scoring, traceability, and report generation. - Add sanitized Codex, OpenCode, and ACPX fixtures. - Add package-boundary and clean-consumer checks. - Add the eval package manifest to the Docker dependency stage. - Add the generated protocol fixture digest without changing the lockfile. ## Verification GitHub Actions must run: - Runner TypeScript and Rust type checks. - Runner unit and protocol tests. - Evaluation kernel tests. - Workflow traceability checks. - Clean-consumer and package-boundary checks. - Repository test, type-check, build, policy, and security gates. No local test command was run. The repository owner requested GitHub-only verification. ## Risks This is a large greenfield review surface with 51 files. The code does not launch a live provider or load credentials. Package and protocol drift fail closed. The workspace lockfile remains under the existing CI-owned process. ## Model Used OpenAI Codex with the GPT-5 agent model. The work used high reasoning, repository inspection, tool use, and parallel code review. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge
34 lines
2.9 KiB
JSON
34 lines
2.9 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://paperclip.dev/schemas/runner-workflow-eval-case.v1.schema.json",
|
|
"title": "RunnerWorkflowEvalCase v1",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["schema", "id", "title", "version", "tags", "providers", "steps", "assertions"],
|
|
"properties": {
|
|
"schema": { "const": "paperclip.runner.workflow-eval-case.v1" },
|
|
"id": { "enum": ["final-response", "rich-activity", "verification-policy", "governed-interaction", "steering-causality", "planning-lifecycle", "review-lifecycle", "delegation-return", "completion-robustness", "restart-recovery", "cancellation-permissions", "trace-lineage"] },
|
|
"title": { "type": "string", "minLength": 1 },
|
|
"version": { "const": 1 },
|
|
"tags": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } },
|
|
"providers": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "enum": ["codex", "opencode", "acpx"] } },
|
|
"steps": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": {
|
|
"oneOf": [
|
|
{ "type": "object", "additionalProperties": false, "required": ["kind", "taskMode"], "properties": { "kind": { "const": "run_start" }, "taskMode": { "enum": ["ask", "execute", "plan"] } } },
|
|
{ "type": "object", "additionalProperties": false, "required": ["kind", "interaction"], "properties": { "kind": { "const": "interaction_response" }, "interaction": { "enum": ["questions", "suggest_tasks", "checkbox", "item_verdicts"] }, "partial": { "type": "boolean" } } },
|
|
{ "type": "object", "additionalProperties": false, "required": ["kind", "delivery"], "properties": { "kind": { "const": "steer" }, "delivery": { "enum": ["queued", "active_turn"] }, "duplicate": { "type": "boolean" } } },
|
|
{ "type": "object", "additionalProperties": false, "required": ["kind", "decision"], "properties": { "kind": { "const": "review_decision" }, "decision": { "enum": ["approve", "reject"] } } },
|
|
{ "type": "object", "additionalProperties": false, "required": ["kind", "decision"], "properties": { "kind": { "const": "permission_decision" }, "decision": { "enum": ["allow_once", "accept_for_session", "deny"] } } },
|
|
{ "type": "object", "additionalProperties": false, "required": ["kind", "actor"], "properties": { "kind": { "const": "cancel" }, "actor": { "enum": ["operator", "control_plane"] } } },
|
|
{ "type": "object", "additionalProperties": false, "required": ["kind", "phase"], "properties": { "kind": { "const": "process_restart" }, "phase": { "enum": ["provider_turn", "semantic_result", "finalization"] } } },
|
|
{ "type": "object", "additionalProperties": false, "required": ["kind", "childProvider"], "properties": { "kind": { "const": "child_completion" }, "childProvider": { "enum": ["codex", "opencode", "acpx"] } } }
|
|
]
|
|
}
|
|
},
|
|
"assertions": { "type": "object" }
|
|
}
|
|
}
|