From a1bd737da6c43e70f95bc699e6f4dce3de97e0e3 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:35:10 -0500 Subject: [PATCH] feat(runner): verify ACPX effective models (#12392) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Each qualified ACPX profile binds one exact model and one reported ACP selector. > - A provider can start with a stale default or report a selector that differs from the canonical model name. > - The runner must select and verify the qualified model before it accepts a billable prompt. > - This pull request adds that verification boundary and normalizes a verified selector for provider-neutral consumers. > - The benefit is fail-closed model identity without enabling the ACPX runtime. ## Linked Issues or Issue Description **Agent or provider** Qualified Pi, Claude, and Codex ACP servers through the internal ACPX driver. **Why this adapter is useful** The runner must not assume that an ACP session uses the requested model. It needs a status check, an exact model selection when required, and a second status check before work can begin. **How the agent is invoked** A later pull request will provide the private ACP runtime control behind this interface. This pull request does not launch a process, add a dependency, register an adapter, or change runtime selection. **Additional context** This pull request is stacked on #12391. Claude's qualified canonical model is `claude-sonnet-5`, while its pinned ACP server reports the stable selector `sonnet`. ## What Changed - Require ACP model status before accepting a qualified runtime. - Select the exact canonical model when the session reports a stale default. - Reapply canonical selection when a qualified profile uses a distinct ACP selector. - Fail closed when status, model selection, or the expected reported selector is unavailable. - Normalize a verified selector back to the canonical model for provider-neutral status consumers. - Add tests for exact, stale, aliased, unavailable, and ignored-selection cases. ## Verification - Runner TypeScript typecheck — passed. - Runner TypeScript tests — passed, including 5 new model-verification tests. - `pnpm -r typecheck` — passed for all applicable workspaces. - `pnpm build` — passed, including runner binary, server, UI, and workspace packages. - Prettier and `git diff --check` — passed. - The diff contains 2 files and does not change `pnpm-lock.yaml`, a workflow, a dependency, a public export, server selection, or UI behavior. ## Risks The main risk is treating a provider alias as proof of the requested model. A distinct selector is accepted only when it is part of the immutable qualified profile, after the runner sends the exact canonical model through ACP and performs a second status check. All other mismatches fail closed. ## Model Used OpenAI Codex with GPT-5 and repository tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either linked an existing public item or described the issue in this PR - [x] I have not referenced internal or instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal task identifier - [x] I have run the affected tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have documented the model admission boundary - [ ] All applicable GitHub Actions are green - [ ] Greptile is 5/5 with every actionable comment resolved - [x] I will address all review findings before requesting merge --- .../drivers/acpx/model-verification.test.ts | 108 ++++++++++++++++++ .../src/drivers/acpx/model-verification.ts | 74 ++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts create mode 100644 packages/paperclip-runner/src/drivers/acpx/model-verification.ts diff --git a/packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts b/packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts new file mode 100644 index 00000000000..cd4cf1317a4 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it, vi } from "vitest"; + +import { resolveQualifiedAcpxProfile } from "./qualified-profiles.js"; +import { requireVerifiedAcpxModel } from "./model-verification.js"; + +describe("ACPX qualified model verification", () => { + it("accepts an exact model already reported by the provider", async () => { + const getStatus = vi.fn(async () => ({ + models: { + currentModelId: "gpt-5.6-sol", + availableModelIds: ["gpt-5.6-sol"], + }, + })); + const setModel = vi.fn(async () => undefined); + + await expect( + requireVerifiedAcpxModel( + { getStatus, setModel }, + resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol"), + ), + ).resolves.toMatchObject({ + models: { currentModelId: "gpt-5.6-sol" }, + }); + expect(setModel).not.toHaveBeenCalled(); + }); + + it("selects Claude's canonical model and normalizes its ACP selector", async () => { + const setModel = vi.fn(async () => undefined); + const getStatus = vi.fn(async () => ({ + models: { + currentModelId: "sonnet", + availableModelIds: ["default", "sonnet", "opus"], + }, + })); + + await expect( + requireVerifiedAcpxModel( + { getStatus, setModel }, + resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"), + ), + ).resolves.toMatchObject({ + models: { + currentModelId: "claude-sonnet-5", + availableModelIds: ["default", "claude-sonnet-5", "opus"], + }, + }); + expect(setModel).toHaveBeenCalledTimes(1); + expect(setModel).toHaveBeenCalledWith("claude-sonnet-5"); + expect(getStatus).toHaveBeenCalledTimes(2); + }); + + it("selects a stale default before accepting an exact-selector profile", async () => { + let selected = false; + const setModel = vi.fn(async () => { + selected = true; + }); + const getStatus = vi.fn(async () => ({ + models: { + currentModelId: selected ? "gpt-5.6-sol" : "default", + availableModelIds: ["default", "gpt-5.6-sol"], + }, + })); + + await requireVerifiedAcpxModel( + { getStatus, setModel }, + resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol"), + ); + expect(setModel).toHaveBeenCalledWith("gpt-5.6-sol"); + }); + + it("fails closed when status or model selection is unavailable", async () => { + const profile = resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol"); + await expect(requireVerifiedAcpxModel({}, profile)).rejects.toThrow( + /cannot verify its effective model/, + ); + await expect( + requireVerifiedAcpxModel( + { + getStatus: async () => ({ + models: { currentModelId: "default", availableModelIds: [] }, + }), + }, + profile, + ), + ).rejects.toThrow(/config options/); + }); + + it("rejects a provider that ignores the qualified model selection", async () => { + const profile = resolveQualifiedAcpxProfile( + "pi", + "openrouter/deepseek/deepseek-v4-flash-0731", + ); + await expect( + requireVerifiedAcpxModel( + { + getStatus: async () => ({ + models: { + currentModelId: "openrouter/other", + availableModelIds: ["openrouter/other"], + }, + }), + setModel: async () => undefined, + }, + profile, + ), + ).rejects.toThrow(/effective model mismatch/); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/acpx/model-verification.ts b/packages/paperclip-runner/src/drivers/acpx/model-verification.ts new file mode 100644 index 00000000000..2eb49ac6e43 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/model-verification.ts @@ -0,0 +1,74 @@ +import type { QualifiedAcpxProfile } from "./qualified-profiles.js"; + +export interface AcpxModelStatus { + models?: { + currentModelId?: string; + availableModelIds?: readonly string[]; + }; + [key: string]: unknown; +} + +export interface AcpxModelControl { + getStatus?: () => Promise; + setModel?: (model: string) => Promise; +} + +/** + * Select and verify the exact qualified model before a billable prompt can be + * accepted. A provider selector is normalized only after ACP reports it. + */ +export async function requireVerifiedAcpxModel( + control: AcpxModelControl, + profile: QualifiedAcpxProfile, +): Promise { + if (!control.getStatus) { + throw new Error("ACPX agent cannot verify its effective model"); + } + const requestedModel = profile.qualificationModel; + let status = await control.getStatus(); + const mustSelectCanonical = + profile.reportedModelId !== requestedModel || + status.models?.currentModelId !== requestedModel; + if (mustSelectCanonical) { + if (!control.setModel) { + throw new Error( + "ACPX agent cannot verify its canonical model through ACP config options", + ); + } + await control.setModel(requestedModel); + status = await control.getStatus(); + } + if (status.models?.currentModelId !== profile.reportedModelId) { + throw new Error( + `ACPX effective model mismatch: requested ${requestedModel}, expected ACP selector ${profile.reportedModelId}, received ${status.models?.currentModelId ?? "unverified"}`, + ); + } + return normalizeQualifiedModelStatus(status, profile); +} + +function normalizeQualifiedModelStatus( + status: AcpxModelStatus, + profile: QualifiedAcpxProfile, +): AcpxModelStatus { + const available = status.models?.availableModelIds ?? []; + const normalizedAvailable = Array.from( + new Set( + available.map((modelId) => + modelId === profile.reportedModelId + ? profile.qualificationModel + : modelId, + ), + ), + ); + if (!normalizedAvailable.includes(profile.qualificationModel)) { + normalizedAvailable.push(profile.qualificationModel); + } + return { + ...status, + models: { + ...status.models, + currentModelId: profile.qualificationModel, + availableModelIds: normalizedAvailable, + }, + }; +}