mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-13 07:12:00 +01:00
## Thinking Path
> - Paperclip is the open source app people use to manage AI agents for
work
> - Each agent has a runtime readiness test ("test environment") that
runs a short hello probe through its harness
> - The OpenCode probe sorts failures into model, timeout,
authentication and generic errors
> - OpenRouter rejects an empty or unknown key with HTTP 401 and the
messages "Missing Authentication header" or "User not found."
> - The probe matched authentication only by wording, so these 401
replies became a generic `opencode_hello_probe_failed` error
> - The server maps `*_hello_probe_auth_required` to the missing-account
state, so the user saw a generic failure instead of "connect an account"
> - This pull request classifies the structured APIError status 401 as
`opencode_hello_probe_auth_required`
> - The benefit is that a member without a working account shows the
missing-account state and its fix
## Linked Issues or Issue Description
**What happened**
An OpenCode (native runner) agent with an empty OpenRouter key failed
its readiness test with the generic error `opencode_hello_probe_failed`.
The chat eval `teams.configure-team` could not form its "one member is
missing an account, one is ready" fixture because of this.
**Expected behavior**
The readiness test reports `opencode_hello_probe_auth_required` (warn)
when the provider rejects authentication with HTTP 401, whatever the
provider's wording.
**Steps to reproduce**
1. Configure a native runner OpenCode agent with model
`openrouter/deepseek/deepseek-v4.1-flash-1017` and `OPENROUTER_API_KEY`
set to an empty string.
2. Run the agent's environment test.
3. OpenCode 1.18.34 prints
`{"type":"error","error":{"name":"APIError","data":{"message":"Missing
Authentication header","statusCode":401}}}`.
4. The result is `opencode_hello_probe_failed` (error), not the
missing-authentication check.
## What Changed
- `parse.ts`: add `openCodeErrorStatusCodes`, which reads the HTTP
status from OpenCode error events.
- `test.ts`: treat a 401 status on the hello probe as
`opencode_hello_probe_auth_required`, in addition to the existing
wording match.
- `test.remote.test.ts`: regression tests with the two recorded
OpenRouter 401 outputs, and a negative control (500 and 429 stay generic
failures).
## Verification
- `pnpm exec vitest run
packages/adapters/opencode-local/src/server/test.remote.test.ts
packages/adapters/opencode-local/src/server/parse.test.ts` passes (28
tests).
- Red check: without the classifier change, the two new 401 tests fail.
- `npx tsc --noEmit -p packages/adapters/opencode-local` passes.
- I recorded the probe output from the pinned OpenCode 1.18.34 binary
against OpenRouter with an empty key and with an unknown key.
## Risks
- Low risk. Only a 401 status changes classification. A 401 means the
provider rejected the credential. Other statuses keep their current
result.
## Model Used
- Anthropic Claude Opus 5.5 (`claude-opus-5-5`), tool use and code
execution in Claude Code.
## Checklist
- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [ ] All Paperclip CI gates are green
- [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
270 lines
14 KiB
TypeScript
270 lines
14 KiB
TypeScript
import { mkdtemp, rm, stat } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
|
|
|
const {
|
|
ensureAdapterExecutionTargetDirectory,
|
|
ensureAdapterExecutionTargetCommandResolvable,
|
|
maybeRunSandboxInstallCommand,
|
|
runAdapterExecutionTargetProcess,
|
|
describeAdapterExecutionTarget,
|
|
resolveAdapterExecutionTargetCwd,
|
|
prepareAdapterExecutionTargetRuntime,
|
|
} = vi.hoisted(() => {
|
|
const restoreWorkspace = vi.fn(async () => {});
|
|
return {
|
|
ensureAdapterExecutionTargetDirectory: vi.fn(async () => {}),
|
|
ensureAdapterExecutionTargetCommandResolvable: vi.fn(async () => {}),
|
|
maybeRunSandboxInstallCommand: vi.fn(async () => null),
|
|
runAdapterExecutionTargetProcess: vi.fn(async () => ({
|
|
exitCode: 0,
|
|
signal: null,
|
|
timedOut: false,
|
|
stdout: [
|
|
JSON.stringify({ type: "step_start", sessionID: "session-1" }),
|
|
JSON.stringify({ type: "text", sessionID: "session-1", part: { text: "hello" } }),
|
|
JSON.stringify({
|
|
type: "step_finish",
|
|
sessionID: "session-1",
|
|
part: { cost: 0.001, tokens: { input: 1, output: 1, reasoning: 0, cache: { read: 0, write: 0 } } },
|
|
}),
|
|
].join("\n"),
|
|
stderr: "",
|
|
pid: 123,
|
|
startedAt: new Date().toISOString(),
|
|
})),
|
|
describeAdapterExecutionTarget: vi.fn(() => "QA Cloudflare"),
|
|
resolveAdapterExecutionTargetCwd: vi.fn((target, configuredCwd, fallbackCwd) => {
|
|
if (typeof configuredCwd === "string" && configuredCwd.trim().length > 0) return configuredCwd;
|
|
if (target && typeof target === "object" && "remoteCwd" in target && typeof target.remoteCwd === "string") {
|
|
return target.remoteCwd;
|
|
}
|
|
return fallbackCwd;
|
|
}),
|
|
prepareAdapterExecutionTargetRuntime: vi.fn(async () => ({
|
|
target: null,
|
|
workspaceRemoteDir: "/remote/workspace/.paperclip-runtime/runs/test/workspace",
|
|
runtimeRootDir: "/remote/workspace/.paperclip-runtime/runs/test/workspace/.paperclip-runtime/opencode",
|
|
assetDirs: {
|
|
xdgConfig: "/remote/workspace/.paperclip-runtime/runs/test/workspace/.paperclip-runtime/opencode/xdgConfig",
|
|
},
|
|
restoreWorkspace,
|
|
})),
|
|
};
|
|
});
|
|
|
|
vi.mock("@paperclipai/adapter-utils/execution-target", async () => {
|
|
const actual = await vi.importActual<typeof import("@paperclipai/adapter-utils/execution-target")>(
|
|
"@paperclipai/adapter-utils/execution-target",
|
|
);
|
|
return {
|
|
...actual,
|
|
ensureAdapterExecutionTargetDirectory,
|
|
ensureAdapterExecutionTargetCommandResolvable,
|
|
maybeRunSandboxInstallCommand,
|
|
runAdapterExecutionTargetProcess,
|
|
describeAdapterExecutionTarget,
|
|
resolveAdapterExecutionTargetCwd,
|
|
prepareAdapterExecutionTargetRuntime,
|
|
};
|
|
});
|
|
|
|
import { testEnvironment } from "./test.js";
|
|
import { discoverOpenCodeModels, ensureOpenCodeModelConfiguredAndAvailable } from "./models.js";
|
|
|
|
vi.mock("./models.js", async () => ({
|
|
...await vi.importActual<typeof import("./models.js")>("./models.js"),
|
|
discoverOpenCodeModels: vi.fn().mockRejectedValue(new Error("catalog should not be needed")),
|
|
ensureOpenCodeModelConfiguredAndAvailable: vi.fn().mockRejectedValue(new Error("catalog should not be needed")),
|
|
}));
|
|
|
|
describe("opencode remote environment diagnostics", () => {
|
|
const configHomes: string[] = [];
|
|
let configHome: string;
|
|
|
|
beforeEach(async () => {
|
|
configHome = await mkdtemp(path.join(os.tmpdir(), "paperclip-opencode-test-config-"));
|
|
vi.stubEnv("XDG_CONFIG_HOME", configHome);
|
|
});
|
|
|
|
afterEach(async () => {
|
|
vi.clearAllMocks();
|
|
vi.unstubAllEnvs();
|
|
await rm(configHome, { recursive: true, force: true });
|
|
await Promise.all(configHomes.splice(0).map(dir => rm(dir, { recursive: true, force: true })));
|
|
});
|
|
|
|
it.each(["passed", "auth_required", "model_unavailable", "timed_out"])(
|
|
"validates a native model with the real probe path and no catalog scan (%s)", async outcome => {
|
|
const successfulProbe = await runAdapterExecutionTargetProcess.getMockImplementation()!();
|
|
runAdapterExecutionTargetProcess.mockResolvedValueOnce(outcome === "passed" ? successfulProbe : {
|
|
...successfulProbe, exitCode: 1, stdout: "", timedOut: outcome === "timed_out",
|
|
stderr: outcome === "auth_required" ? "invalid API key" : "ProviderModelNotFoundError",
|
|
});
|
|
const result = await testEnvironment({ companyId: "company-1", adapterType: "paperclip_runner",
|
|
config: { model: "openrouter/deepseek/deepseek-v4-flash-0731", env: { XDG_CONFIG_HOME: configHome } } });
|
|
expect(discoverOpenCodeModels).not.toHaveBeenCalled();
|
|
expect(ensureOpenCodeModelConfiguredAndAvailable).not.toHaveBeenCalled();
|
|
expect(runAdapterExecutionTargetProcess).toHaveBeenCalledExactlyOnceWith(
|
|
expect.any(String), null, "opencode", ["run", "--format", "json", "--model", "openrouter/deepseek/deepseek-v4-flash-0731"],
|
|
expect.objectContaining({ stdin: "Respond with hello." }),
|
|
);
|
|
expect(result.status).toBe(outcome === "passed" ? "pass" : "warn");
|
|
expect(result.checks).toContainEqual(expect.objectContaining({ code: `opencode_hello_probe_${outcome}` }));
|
|
const call = runAdapterExecutionTargetProcess.mock.calls[0] as unknown as [unknown, unknown, unknown, unknown, { cwd: string }];
|
|
expect(path.basename(call[4].cwd)).toMatch(/^paperclip-opencode-native-probe-/);
|
|
await expect(stat(call[4].cwd)).rejects.toMatchObject({ code: "ENOENT" });
|
|
},
|
|
);
|
|
|
|
it.each(["Missing Authentication header", "User not found."])(
|
|
"reports a provider 401 as missing authentication, not a generic failure (%s)", async message => {
|
|
const successfulProbe = await runAdapterExecutionTargetProcess.getMockImplementation()!();
|
|
runAdapterExecutionTargetProcess.mockResolvedValueOnce({ ...successfulProbe, exitCode: 1, stderr: "",
|
|
stdout: JSON.stringify({ type: "error", sessionID: "session-1", error: { name: "APIError",
|
|
data: { message, statusCode: 401, isRetryable: false, responseHeaders: { "access-control-expose-headers": "WWW-Authenticate" } } } }) });
|
|
const result = await testEnvironment({ companyId: "company-1", adapterType: "paperclip_runner",
|
|
config: { model: "openrouter/deepseek/deepseek-v4-flash-0731", env: { XDG_CONFIG_HOME: configHome } } });
|
|
expect(result.status).toBe("warn");
|
|
expect(result.checks.map(check => [check.code, check.level])).toContainEqual(["opencode_hello_probe_auth_required", "warn"]);
|
|
expect(result.checks.some(check => check.level === "error")).toBe(false);
|
|
},
|
|
);
|
|
|
|
it.each([500, 429])("keeps other provider statuses a generic probe failure (%s)", async statusCode => {
|
|
const successfulProbe = await runAdapterExecutionTargetProcess.getMockImplementation()!();
|
|
runAdapterExecutionTargetProcess.mockResolvedValueOnce({ ...successfulProbe, exitCode: 1, stderr: "",
|
|
stdout: JSON.stringify({ type: "error", error: { name: "APIError", data: { message: "Upstream error", statusCode } } }) });
|
|
const result = await testEnvironment({ companyId: "company-1", adapterType: "paperclip_runner",
|
|
config: { model: "openrouter/deepseek/deepseek-v4-flash-0731", env: { XDG_CONFIG_HOME: configHome } } });
|
|
expect(result.checks).toContainEqual(expect.objectContaining({ code: "opencode_hello_probe_failed", level: "error" }));
|
|
});
|
|
|
|
it("gives each concurrent native probe its own OpenCode data directory", async () => {
|
|
vi.stubEnv("HOME", "/host/user");
|
|
vi.stubEnv("XDG_DATA_HOME", "/host/user/.local/share");
|
|
const probe = () => testEnvironment({ companyId: "company-1", adapterType: "paperclip_runner",
|
|
config: { model: "openrouter/deepseek/deepseek-v4-flash-0731", env: { XDG_CONFIG_HOME: configHome } } });
|
|
const results = await Promise.all([probe(), probe()]);
|
|
expect(results.map(result => result.status)).toEqual(["pass", "pass"]);
|
|
const envs = runAdapterExecutionTargetProcess.mock.calls.map(call =>
|
|
(call as unknown as [unknown, unknown, unknown, unknown, { env: Record<string, string> }])[4].env);
|
|
expect(envs).toHaveLength(2);
|
|
for (const env of envs) {
|
|
expect(path.basename(env.HOME)).toMatch(/^paperclip-opencode-native-probe-home-/);
|
|
for (const key of ["XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME"]) {
|
|
expect(path.dirname(env[key])).toBe(env.HOME);
|
|
}
|
|
expect(env).toMatchObject({ OPENCODE_DISABLE_MODELS_FETCH: "true", OPENCODE_DISABLE_DEFAULT_PLUGINS: "true" });
|
|
await expect(stat(env.HOME)).rejects.toMatchObject({ code: "ENOENT" });
|
|
}
|
|
expect(envs[0].XDG_DATA_HOME).not.toBe(envs[1].XDG_DATA_HOME);
|
|
});
|
|
|
|
it("keeps the host OpenCode home for the legacy local adapter, which owns its login", async () => {
|
|
vi.stubEnv("HOME", "/host/user");
|
|
vi.stubEnv("XDG_DATA_HOME", "/host/user/.local/share");
|
|
vi.mocked(discoverOpenCodeModels).mockResolvedValueOnce([{ id: "provider/model", label: "Model" }]);
|
|
vi.mocked(ensureOpenCodeModelConfiguredAndAvailable).mockResolvedValueOnce([{ id: "provider/model", label: "Model" }]);
|
|
await testEnvironment({ companyId: "company-1", adapterType: "opencode_local",
|
|
config: { model: "provider/model", env: { XDG_CONFIG_HOME: configHome } } });
|
|
const call = runAdapterExecutionTargetProcess.mock.calls[0] as unknown as [unknown, unknown, unknown, unknown, { env: Record<string, string> }];
|
|
expect(call[4].env).toMatchObject({ HOME: "/host/user", XDG_DATA_HOME: "/host/user/.local/share" });
|
|
});
|
|
|
|
it("rejects a malformed native model before starting the probe", async () => {
|
|
const result = await testEnvironment({ companyId: "company-1", adapterType: "paperclip_runner",
|
|
config: { model: "missing-provider", env: { XDG_CONFIG_HOME: configHome } } });
|
|
expect(result.status).toBe("fail");
|
|
expect(runAdapterExecutionTargetProcess).not.toHaveBeenCalled();
|
|
expect(discoverOpenCodeModels).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("preserves an explicitly configured native probe directory", async () => {
|
|
await testEnvironment({ companyId: "company-1", adapterType: "paperclip_runner",
|
|
config: { model: "provider/model", cwd: configHome, env: { XDG_CONFIG_HOME: configHome } } });
|
|
const call = runAdapterExecutionTargetProcess.mock.calls[0] as unknown as [unknown, unknown, unknown, unknown, { cwd: string }];
|
|
expect(call[4].cwd).toBe(configHome);
|
|
expect((await stat(configHome)).isDirectory()).toBe(true);
|
|
});
|
|
|
|
it("keeps catalog validation for the legacy local adapter", async () => {
|
|
vi.mocked(discoverOpenCodeModels).mockResolvedValueOnce([{ id: "provider/model", label: "Model" }]);
|
|
vi.mocked(ensureOpenCodeModelConfiguredAndAvailable).mockResolvedValueOnce([{ id: "provider/model", label: "Model" }]);
|
|
const result = await testEnvironment({ companyId: "company-1", adapterType: "opencode_local",
|
|
config: { model: "provider/model", env: { XDG_CONFIG_HOME: configHome } } });
|
|
expect(result.status).toBe("pass");
|
|
expect(discoverOpenCodeModels).toHaveBeenCalledOnce();
|
|
expect(ensureOpenCodeModelConfiguredAndAvailable).toHaveBeenCalledOnce();
|
|
expect(runAdapterExecutionTargetProcess).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it.each([false, true])("stages remote runtime config assets for sandbox hello probes (managed=%s)", async (managed) => {
|
|
const configHome = await mkdtemp(path.join(os.tmpdir(), "opencode-remote-test-config-"));
|
|
configHomes.push(configHome);
|
|
const remoteTarget: AdapterExecutionTarget = {
|
|
kind: "remote",
|
|
transport: "sandbox",
|
|
providerKey: "cloudflare",
|
|
remoteCwd: "/remote/workspace",
|
|
runner: {
|
|
execute: async () => ({
|
|
exitCode: 0,
|
|
signal: null,
|
|
timedOut: false,
|
|
stdout: "",
|
|
stderr: "",
|
|
pid: null,
|
|
startedAt: new Date().toISOString(),
|
|
}),
|
|
},
|
|
};
|
|
|
|
const result = await testEnvironment({
|
|
companyId: "company-1",
|
|
adapterType: "opencode_local",
|
|
config: {
|
|
command: "opencode",
|
|
model: "anthropic/claude-sonnet-4-5",
|
|
...(managed ? {
|
|
managedAiConnection: { provider: "openrouter", method: "api_key" },
|
|
} : {}),
|
|
env: {
|
|
XDG_CONFIG_HOME: configHome,
|
|
...(managed ? { OPENAI_API_KEY: "", OPENROUTER_API_KEY: "fixture", HOME: "/var/folders/qa-managed", XDG_DATA_HOME: "/var/folders/qa-managed/data" } : {}),
|
|
},
|
|
},
|
|
executionTarget: remoteTarget,
|
|
environmentName: "QA Cloudflare",
|
|
});
|
|
|
|
expect(result.status).toBe("pass");
|
|
expect(prepareAdapterExecutionTargetRuntime).toHaveBeenCalledTimes(1);
|
|
const runtimeCalls = prepareAdapterExecutionTargetRuntime.mock.calls as unknown as Array<
|
|
[{ adapterKey: string; assets?: Array<{ key: string; localDir: string }> }]
|
|
>;
|
|
const runtimeInput = runtimeCalls[0]?.[0];
|
|
expect(runtimeInput?.adapterKey).toBe("opencode");
|
|
expect(runtimeInput?.assets).toEqual([
|
|
expect.objectContaining({
|
|
key: "xdgConfig",
|
|
}),
|
|
]);
|
|
|
|
const probeCall = runAdapterExecutionTargetProcess.mock.calls[0] as unknown as
|
|
| [string, AdapterExecutionTarget, string, string[], { cwd: string; env: Record<string, string> }]
|
|
| undefined;
|
|
expect(probeCall?.[4].cwd).toBe("/remote/workspace/.paperclip-runtime/runs/test/workspace");
|
|
if (managed) {
|
|
expect(probeCall?.[4].env.HOME).toContain("/remote/workspace/.paperclip-runtime/runs/test/workspace/.paperclip-runtime/opencode/managed-auth/");
|
|
expect(probeCall?.[4].env.XDG_DATA_HOME).toBe(`${probeCall?.[4].env.HOME}/data`);
|
|
expect(probeCall?.[4].env.XDG_CACHE_HOME).toBe(`${probeCall?.[4].env.HOME}/cache`);
|
|
}
|
|
expect(probeCall?.[4].env.XDG_CONFIG_HOME).toBe(
|
|
"/remote/workspace/.paperclip-runtime/runs/test/workspace/.paperclip-runtime/opencode/xdgConfig",
|
|
);
|
|
});
|
|
});
|