feat(runner): verify ACPX effective models (#12392)

## Thinking Path

> - Paperclip is the open source app people use to manage AI agents for
work.
> - Each qualified ACPX profile binds one exact model and one reported
ACP selector.
> - A provider can start with a stale default or report a selector that
differs from the canonical model name.
> - The runner must select and verify the qualified model before it
accepts a billable prompt.
> - This pull request adds that verification boundary and normalizes a
verified selector for provider-neutral consumers.
> - The benefit is fail-closed model identity without enabling the ACPX
runtime.

## Linked Issues or Issue Description

**Agent or provider**

Qualified Pi, Claude, and Codex ACP servers through the internal ACPX
driver.

**Why this adapter is useful**

The runner must not assume that an ACP session uses the requested model.
It needs a status check, an exact model selection when required, and a
second status check before work can begin.

**How the agent is invoked**

A later pull request will provide the private ACP runtime control behind
this interface. This pull request does not launch a process, add a
dependency, register an adapter, or change runtime selection.

**Additional context**

This pull request is stacked on #12391. Claude's qualified canonical
model is `claude-sonnet-5`, while its pinned ACP server reports the
stable selector `sonnet`.

## What Changed

- Require ACP model status before accepting a qualified runtime.
- Select the exact canonical model when the session reports a stale
default.
- Reapply canonical selection when a qualified profile uses a distinct
ACP selector.
- Fail closed when status, model selection, or the expected reported
selector is unavailable.
- Normalize a verified selector back to the canonical model for
provider-neutral status consumers.
- Add tests for exact, stale, aliased, unavailable, and
ignored-selection cases.

## Verification

- Runner TypeScript typecheck — passed.
- Runner TypeScript tests — passed, including 5 new model-verification
tests.
- `pnpm -r typecheck` — passed for all applicable workspaces.
- `pnpm build` — passed, including runner binary, server, UI, and
workspace packages.
- Prettier and `git diff --check` — passed.
- The diff contains 2 files and does not change `pnpm-lock.yaml`, a
workflow, a dependency, a public export, server selection, or UI
behavior.

## Risks

The main risk is treating a provider alias as proof of the requested
model. A distinct selector is accepted only when it is part of the
immutable qualified profile, after the runner sends the exact canonical
model through ACP and performs a second status check. All other
mismatches fail closed.

## Model Used

OpenAI Codex with GPT-5 and repository tool use.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either linked an existing public item or described the
issue in this PR
- [x] I have not referenced internal or instance-local Paperclip issues
or links
- [x] My branch name describes the change and contains no internal task
identifier
- [x] I have run the affected tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have documented the model admission boundary
- [ ] All applicable GitHub Actions are green
- [ ] Greptile is 5/5 with every actionable comment resolved
- [x] I will address all review findings before requesting merge
This commit is contained in:
Dotta authored and GitHub committed 2026-08-30 13:35:10 -05:00
1 parent 07a80cda41
commit a1bd737da6
2 files changed
+182

No files matched your search

@@ -0,0 +1,108 @@
import { describe, expect, it, vi } from "vitest";
import { resolveQualifiedAcpxProfile } from "./qualified-profiles.js";
import { requireVerifiedAcpxModel } from "./model-verification.js";
describe("ACPX qualified model verification", () => {
it("accepts an exact model already reported by the provider", async () => {
const getStatus = vi.fn(async () => ({
models: {
currentModelId: "gpt-5.6-sol",
availableModelIds: ["gpt-5.6-sol"],
},
}));
const setModel = vi.fn(async () => undefined);
await expect(
requireVerifiedAcpxModel(
{ getStatus, setModel },
resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol"),
),
).resolves.toMatchObject({
models: { currentModelId: "gpt-5.6-sol" },
});
expect(setModel).not.toHaveBeenCalled();
});
it("selects Claude's canonical model and normalizes its ACP selector", async () => {
const setModel = vi.fn(async () => undefined);
const getStatus = vi.fn(async () => ({
models: {
currentModelId: "sonnet",
availableModelIds: ["default", "sonnet", "opus"],
},
}));
await expect(
requireVerifiedAcpxModel(
{ getStatus, setModel },
resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"),
),
).resolves.toMatchObject({
models: {
currentModelId: "claude-sonnet-5",
availableModelIds: ["default", "claude-sonnet-5", "opus"],
},
});
expect(setModel).toHaveBeenCalledTimes(1);
expect(setModel).toHaveBeenCalledWith("claude-sonnet-5");
expect(getStatus).toHaveBeenCalledTimes(2);
});
it("selects a stale default before accepting an exact-selector profile", async () => {
let selected = false;
const setModel = vi.fn(async () => {
selected = true;
});
const getStatus = vi.fn(async () => ({
models: {
currentModelId: selected ? "gpt-5.6-sol" : "default",
availableModelIds: ["default", "gpt-5.6-sol"],
},
}));
await requireVerifiedAcpxModel(
{ getStatus, setModel },
resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol"),
);
expect(setModel).toHaveBeenCalledWith("gpt-5.6-sol");
});
it("fails closed when status or model selection is unavailable", async () => {
const profile = resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol");
await expect(requireVerifiedAcpxModel({}, profile)).rejects.toThrow(
/cannot verify its effective model/,
);
await expect(
requireVerifiedAcpxModel(
{
getStatus: async () => ({
models: { currentModelId: "default", availableModelIds: [] },
}),
},
profile,
),
).rejects.toThrow(/config options/);
});
it("rejects a provider that ignores the qualified model selection", async () => {
const profile = resolveQualifiedAcpxProfile(
"pi",
"openrouter/deepseek/deepseek-v4-flash-0731",
);
await expect(
requireVerifiedAcpxModel(
{
getStatus: async () => ({
models: {
currentModelId: "openrouter/other",
availableModelIds: ["openrouter/other"],
},
}),
setModel: async () => undefined,
},
profile,
),
).rejects.toThrow(/effective model mismatch/);
});
});
@@ -0,0 +1,74 @@
import type { QualifiedAcpxProfile } from "./qualified-profiles.js";
export interface AcpxModelStatus {
models?: {
currentModelId?: string;
availableModelIds?: readonly string[];
};
[key: string]: unknown;
}
export interface AcpxModelControl {
getStatus?: () => Promise<AcpxModelStatus>;
setModel?: (model: string) => Promise<void>;
}
/**
* Select and verify the exact qualified model before a billable prompt can be
* accepted. A provider selector is normalized only after ACP reports it.
*/
export async function requireVerifiedAcpxModel(
control: AcpxModelControl,
profile: QualifiedAcpxProfile,
): Promise<AcpxModelStatus> {
if (!control.getStatus) {
throw new Error("ACPX agent cannot verify its effective model");
}
const requestedModel = profile.qualificationModel;
let status = await control.getStatus();
const mustSelectCanonical =
profile.reportedModelId !== requestedModel ||
status.models?.currentModelId !== requestedModel;
if (mustSelectCanonical) {
if (!control.setModel) {
throw new Error(
"ACPX agent cannot verify its canonical model through ACP config options",
);
}
await control.setModel(requestedModel);
status = await control.getStatus();
}
if (status.models?.currentModelId !== profile.reportedModelId) {
throw new Error(
`ACPX effective model mismatch: requested ${requestedModel}, expected ACP selector ${profile.reportedModelId}, received ${status.models?.currentModelId ?? "unverified"}`,
);
}
return normalizeQualifiedModelStatus(status, profile);
}
function normalizeQualifiedModelStatus(
status: AcpxModelStatus,
profile: QualifiedAcpxProfile,
): AcpxModelStatus {
const available = status.models?.availableModelIds ?? [];
const normalizedAvailable = Array.from(
new Set(
available.map((modelId) =>
modelId === profile.reportedModelId
? profile.qualificationModel
: modelId,
),
),
);
if (!normalizedAvailable.includes(profile.qualificationModel)) {
normalizedAvailable.push(profile.qualificationModel);
}
return {
...status,
models: {
...status.models,
currentModelId: profile.qualificationModel,
availableModelIds: normalizedAvailable,
},
};
}