mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-29 12:59:13 +01:00
feat(runner): verify ACPX effective models (#12392)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Each qualified ACPX profile binds one exact model and one reported ACP selector. > - A provider can start with a stale default or report a selector that differs from the canonical model name. > - The runner must select and verify the qualified model before it accepts a billable prompt. > - This pull request adds that verification boundary and normalizes a verified selector for provider-neutral consumers. > - The benefit is fail-closed model identity without enabling the ACPX runtime. ## Linked Issues or Issue Description **Agent or provider** Qualified Pi, Claude, and Codex ACP servers through the internal ACPX driver. **Why this adapter is useful** The runner must not assume that an ACP session uses the requested model. It needs a status check, an exact model selection when required, and a second status check before work can begin. **How the agent is invoked** A later pull request will provide the private ACP runtime control behind this interface. This pull request does not launch a process, add a dependency, register an adapter, or change runtime selection. **Additional context** This pull request is stacked on #12391. Claude's qualified canonical model is `claude-sonnet-5`, while its pinned ACP server reports the stable selector `sonnet`. ## What Changed - Require ACP model status before accepting a qualified runtime. - Select the exact canonical model when the session reports a stale default. - Reapply canonical selection when a qualified profile uses a distinct ACP selector. - Fail closed when status, model selection, or the expected reported selector is unavailable. - Normalize a verified selector back to the canonical model for provider-neutral status consumers. - Add tests for exact, stale, aliased, unavailable, and ignored-selection cases. ## Verification - Runner TypeScript typecheck — passed. - Runner TypeScript tests — passed, including 5 new model-verification tests. - `pnpm -r typecheck` — passed for all applicable workspaces. - `pnpm build` — passed, including runner binary, server, UI, and workspace packages. - Prettier and `git diff --check` — passed. - The diff contains 2 files and does not change `pnpm-lock.yaml`, a workflow, a dependency, a public export, server selection, or UI behavior. ## Risks The main risk is treating a provider alias as proof of the requested model. A distinct selector is accepted only when it is part of the immutable qualified profile, after the runner sends the exact canonical model through ACP and performs a second status check. All other mismatches fail closed. ## Model Used OpenAI Codex with GPT-5 and repository tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either linked an existing public item or described the issue in this PR - [x] I have not referenced internal or instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal task identifier - [x] I have run the affected tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have documented the model admission boundary - [ ] All applicable GitHub Actions are green - [ ] Greptile is 5/5 with every actionable comment resolved - [x] I will address all review findings before requesting merge
This commit is contained in:
1 parent
07a80cda41
commit
a1bd737da6
2 files changed
+182
No files matched your search
@@ -0,0 +1,108 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { resolveQualifiedAcpxProfile } from "./qualified-profiles.js";
|
||||
import { requireVerifiedAcpxModel } from "./model-verification.js";
|
||||
|
||||
describe("ACPX qualified model verification", () => {
|
||||
it("accepts an exact model already reported by the provider", async () => {
|
||||
const getStatus = vi.fn(async () => ({
|
||||
models: {
|
||||
currentModelId: "gpt-5.6-sol",
|
||||
availableModelIds: ["gpt-5.6-sol"],
|
||||
},
|
||||
}));
|
||||
const setModel = vi.fn(async () => undefined);
|
||||
|
||||
await expect(
|
||||
requireVerifiedAcpxModel(
|
||||
{ getStatus, setModel },
|
||||
resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol"),
|
||||
),
|
||||
).resolves.toMatchObject({
|
||||
models: { currentModelId: "gpt-5.6-sol" },
|
||||
});
|
||||
expect(setModel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("selects Claude's canonical model and normalizes its ACP selector", async () => {
|
||||
const setModel = vi.fn(async () => undefined);
|
||||
const getStatus = vi.fn(async () => ({
|
||||
models: {
|
||||
currentModelId: "sonnet",
|
||||
availableModelIds: ["default", "sonnet", "opus"],
|
||||
},
|
||||
}));
|
||||
|
||||
await expect(
|
||||
requireVerifiedAcpxModel(
|
||||
{ getStatus, setModel },
|
||||
resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"),
|
||||
),
|
||||
).resolves.toMatchObject({
|
||||
models: {
|
||||
currentModelId: "claude-sonnet-5",
|
||||
availableModelIds: ["default", "claude-sonnet-5", "opus"],
|
||||
},
|
||||
});
|
||||
expect(setModel).toHaveBeenCalledTimes(1);
|
||||
expect(setModel).toHaveBeenCalledWith("claude-sonnet-5");
|
||||
expect(getStatus).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("selects a stale default before accepting an exact-selector profile", async () => {
|
||||
let selected = false;
|
||||
const setModel = vi.fn(async () => {
|
||||
selected = true;
|
||||
});
|
||||
const getStatus = vi.fn(async () => ({
|
||||
models: {
|
||||
currentModelId: selected ? "gpt-5.6-sol" : "default",
|
||||
availableModelIds: ["default", "gpt-5.6-sol"],
|
||||
},
|
||||
}));
|
||||
|
||||
await requireVerifiedAcpxModel(
|
||||
{ getStatus, setModel },
|
||||
resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol"),
|
||||
);
|
||||
expect(setModel).toHaveBeenCalledWith("gpt-5.6-sol");
|
||||
});
|
||||
|
||||
it("fails closed when status or model selection is unavailable", async () => {
|
||||
const profile = resolveQualifiedAcpxProfile("codex", "gpt-5.6-sol");
|
||||
await expect(requireVerifiedAcpxModel({}, profile)).rejects.toThrow(
|
||||
/cannot verify its effective model/,
|
||||
);
|
||||
await expect(
|
||||
requireVerifiedAcpxModel(
|
||||
{
|
||||
getStatus: async () => ({
|
||||
models: { currentModelId: "default", availableModelIds: [] },
|
||||
}),
|
||||
},
|
||||
profile,
|
||||
),
|
||||
).rejects.toThrow(/config options/);
|
||||
});
|
||||
|
||||
it("rejects a provider that ignores the qualified model selection", async () => {
|
||||
const profile = resolveQualifiedAcpxProfile(
|
||||
"pi",
|
||||
"openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
);
|
||||
await expect(
|
||||
requireVerifiedAcpxModel(
|
||||
{
|
||||
getStatus: async () => ({
|
||||
models: {
|
||||
currentModelId: "openrouter/other",
|
||||
availableModelIds: ["openrouter/other"],
|
||||
},
|
||||
}),
|
||||
setModel: async () => undefined,
|
||||
},
|
||||
profile,
|
||||
),
|
||||
).rejects.toThrow(/effective model mismatch/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { QualifiedAcpxProfile } from "./qualified-profiles.js";
|
||||
|
||||
export interface AcpxModelStatus {
|
||||
models?: {
|
||||
currentModelId?: string;
|
||||
availableModelIds?: readonly string[];
|
||||
};
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface AcpxModelControl {
|
||||
getStatus?: () => Promise<AcpxModelStatus>;
|
||||
setModel?: (model: string) => Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Select and verify the exact qualified model before a billable prompt can be
|
||||
* accepted. A provider selector is normalized only after ACP reports it.
|
||||
*/
|
||||
export async function requireVerifiedAcpxModel(
|
||||
control: AcpxModelControl,
|
||||
profile: QualifiedAcpxProfile,
|
||||
): Promise<AcpxModelStatus> {
|
||||
if (!control.getStatus) {
|
||||
throw new Error("ACPX agent cannot verify its effective model");
|
||||
}
|
||||
const requestedModel = profile.qualificationModel;
|
||||
let status = await control.getStatus();
|
||||
const mustSelectCanonical =
|
||||
profile.reportedModelId !== requestedModel ||
|
||||
status.models?.currentModelId !== requestedModel;
|
||||
if (mustSelectCanonical) {
|
||||
if (!control.setModel) {
|
||||
throw new Error(
|
||||
"ACPX agent cannot verify its canonical model through ACP config options",
|
||||
);
|
||||
}
|
||||
await control.setModel(requestedModel);
|
||||
status = await control.getStatus();
|
||||
}
|
||||
if (status.models?.currentModelId !== profile.reportedModelId) {
|
||||
throw new Error(
|
||||
`ACPX effective model mismatch: requested ${requestedModel}, expected ACP selector ${profile.reportedModelId}, received ${status.models?.currentModelId ?? "unverified"}`,
|
||||
);
|
||||
}
|
||||
return normalizeQualifiedModelStatus(status, profile);
|
||||
}
|
||||
|
||||
function normalizeQualifiedModelStatus(
|
||||
status: AcpxModelStatus,
|
||||
profile: QualifiedAcpxProfile,
|
||||
): AcpxModelStatus {
|
||||
const available = status.models?.availableModelIds ?? [];
|
||||
const normalizedAvailable = Array.from(
|
||||
new Set(
|
||||
available.map((modelId) =>
|
||||
modelId === profile.reportedModelId
|
||||
? profile.qualificationModel
|
||||
: modelId,
|
||||
),
|
||||
),
|
||||
);
|
||||
if (!normalizedAvailable.includes(profile.qualificationModel)) {
|
||||
normalizedAvailable.push(profile.qualificationModel);
|
||||
}
|
||||
return {
|
||||
...status,
|
||||
models: {
|
||||
...status.models,
|
||||
currentModelId: profile.qualificationModel,
|
||||
availableModelIds: normalizedAvailable,
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user