mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
109 lines
8.7 KiB
TypeScript
109 lines
8.7 KiB
TypeScript
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import type { ServerAdapterModule } from "../../server/src/adapters/types.js";
|
|
import { installChatReadinessObserver, readChatReadinessReceipts, readinessErrorCategory } from "./chat-readiness-observer.js";
|
|
const roots: string[] = [];
|
|
const companyId = "11111111-1111-4111-8111-111111111111";
|
|
const ids = ["chat-features.runner-opencode.local.create-backlog"];
|
|
const context = { companyId, adapterType: "paperclip_runner", config: { provider: "opencode", env: { PRIVATE_KEY: "private-config-canary" } } };
|
|
const result = () => ({ adapterType: "paperclip_runner", status: "fail" as const, testedAt: "2026-10-27T14:00:00Z",
|
|
checks: [{ code: "opencode_hello_probe_failed", level: "error" as const, message: "OpenCode hello probe failed.",
|
|
detail: "HTTP 429 private-provider-canary", hint: "private-hint-canary" }] });
|
|
async function fixture() {
|
|
const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "readiness-observer-"))); roots.push(temporaryRoot);
|
|
const testEnvironment = vi.fn(async () => result());
|
|
const adapter = { testEnvironment } as Pick<ServerAdapterModule, "testEnvironment">;
|
|
await installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter });
|
|
return { temporaryRoot, adapter, testEnvironment, receipts: () => readChatReadinessReceipts(temporaryRoot, companyId) };
|
|
}
|
|
afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
|
|
describe("transparent same-call readiness retention", () => {
|
|
it("calls exactly once with the original args/this and returns the identical result", async () => {
|
|
const f = await fixture(), original = result(); f.testEnvironment.mockResolvedValue(original);
|
|
expect(await f.adapter.testEnvironment(context)).toBe(original);
|
|
expect(f.testEnvironment).toHaveBeenCalledExactlyOnceWith(context);
|
|
expect(f.testEnvironment.mock.contexts[0]).toBe(f.adapter);
|
|
const receipts = await f.receipts();
|
|
expect(receipts).toMatchObject([{ companyId, sequence: 1, status: "fail", thrown: false,
|
|
checks: [{ code: "opencode_hello_probe_failed", level: "error", category: "rate-limit" }] }]);
|
|
const bytes = await readFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), "utf8");
|
|
expect(bytes + JSON.stringify(receipts)).not.toMatch(/private-|detail|message|hint|env|config/);
|
|
expect(receipts[0]).not.toHaveProperty("agentId");
|
|
});
|
|
it("accepts the existing launcher directory beneath its private root", async () => {
|
|
const temporaryRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "readiness-observer-"))); roots.push(temporaryRoot);
|
|
await mkdir(path.join(temporaryRoot, "artifacts-private"), { mode: 0o755 });
|
|
const adapter: Pick<ServerAdapterModule, "testEnvironment"> = { testEnvironment: vi.fn(async () => result()) };
|
|
await installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter });
|
|
await adapter.testEnvironment(context);
|
|
expect(await readChatReadinessReceipts(temporaryRoot, companyId)).toHaveLength(1);
|
|
await chmod(temporaryRoot, 0o755);
|
|
await expect(installChatReadinessObserver({ executionIds: ids, temporaryRoot, adapter })).rejects.toThrow("owned private attempt root");
|
|
});
|
|
it("rethrows the identical error without another call or raw error output", async () => {
|
|
const f = await fixture(), error = new Error("Cannot find package private-package-canary");
|
|
f.testEnvironment.mockRejectedValue(error);
|
|
await expect(f.adapter.testEnvironment(context)).rejects.toBe(error);
|
|
expect(f.testEnvironment).toHaveBeenCalledTimes(1);
|
|
expect(await f.receipts()).toMatchObject([{ status: "unknown", thrown: true, errorCategory: "module-missing" }]);
|
|
expect(JSON.stringify(await f.receipts())).not.toContain("private-package-canary");
|
|
});
|
|
it("never replaces the original result when diagnostic storage fails", async () => {
|
|
const f = await fixture(), original = result(); f.testEnvironment.mockResolvedValue(original);
|
|
await writeFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), "", { mode: 0o644 });
|
|
expect(await f.adapter.testEnvironment(context)).toBe(original);
|
|
expect(f.testEnvironment).toHaveBeenCalledTimes(1);
|
|
});
|
|
it("keeps sequence only within the attempt and filters foreign companies", async () => {
|
|
const f = await fixture(); await f.adapter.testEnvironment(context); await f.adapter.testEnvironment(context);
|
|
expect((await f.receipts()).map(row => row.sequence)).toEqual([1, 2]);
|
|
expect(await readChatReadinessReceipts(f.temporaryRoot, "22222222-2222-4222-8222-222222222222")).toEqual([]);
|
|
});
|
|
it("reprojects arbitrary keys, status, codes and categories as unknown", async () => {
|
|
const f = await fixture();
|
|
await writeFile(path.join(f.temporaryRoot, "artifacts-private", "chat-readiness-probes.jsonl"), JSON.stringify({ schema: "paperclip.chat-readiness-probe/v1", companyId, sequence: 1,
|
|
status: "private-status", secret: "private-extra", checks: [{ code: "private-code", level: "private-level", category: "private-category", detail: "private-detail" }] }));
|
|
const receipts = await f.receipts();
|
|
expect(receipts).toMatchObject([{ status: "unknown", checks: [{ code: "unknown", level: "unknown", category: "unknown" }] }]);
|
|
expect(JSON.stringify(receipts)).not.toContain("private-");
|
|
});
|
|
it("does not observe an unrelated provider or execution", async () => {
|
|
const f = await fixture(); await f.adapter.testEnvironment({ ...context, config: { provider: "codex" } });
|
|
expect(await f.receipts()).toEqual([]);
|
|
const original = vi.fn(async () => result()), adapter = { testEnvironment: original };
|
|
await installChatReadinessObserver({ executionIds: ["other.runner-opencode.local.case"], temporaryRoot: "", adapter });
|
|
expect(adapter.testEnvironment).toBe(original);
|
|
});
|
|
it.each([
|
|
["agent_runner_config_incompatible", "unknown"], ["paperclip_runner_mode_invalid", "unknown"], ["paperclip_runner_opencode_model_invalid", "unknown"],
|
|
])("preserves source profile-validation code %s without its message", async (code, category) => {
|
|
const f = await fixture(); f.testEnvironment.mockResolvedValue({ ...result(), checks: [{ code, level: "error", message: "private-profile-canary", detail: "private-canary", hint: "private-canary" }] });
|
|
await f.adapter.testEnvironment(context);
|
|
expect((await f.receipts())[0]!.checks).toEqual([{ code, level: "error", category }]);
|
|
expect(JSON.stringify(await f.receipts())).not.toContain("private-");
|
|
});
|
|
// Actual check.message/detail layouts emitted by opencode-local/server/test.ts:
|
|
// summarizeProbeDetail selects parsed error, first stderr or first stdout line.
|
|
it.each([
|
|
["opencode_command_unresolvable", "Command is not executable: ENOENT", "opencode", "command-missing"],
|
|
["opencode_openai_api_key_missing", "OPENAI_API_KEY override is empty.", "", "unknown"],
|
|
["opencode_hello_probe_auth_required", "OpenCode is installed, but provider authentication is not ready.", "HTTP 401 Unauthorized private-canary", "authentication"],
|
|
["opencode_hello_probe_model_unavailable", "The configured model was not found by the provider.", "ProviderModelNotFoundError private-canary", "model-unavailable"],
|
|
["opencode_hello_probe_failed", "OpenCode hello probe failed.", '{"name":"APIError","data":{"statusCode":429,"message":"private-canary"}}', "rate-limit"],
|
|
["opencode_hello_probe_failed", "OpenCode hello probe failed.", "fetch failed ECONNREFUSED private-canary", "network"],
|
|
["opencode_hello_probe_timed_out", "OpenCode hello probe timed out.", "", "timeout"],
|
|
["opencode_hello_probe_failed", "OpenCode hello probe failed.", "private-unknown-canary", "unknown"],
|
|
])("retains closed actual-source diagnostic shape %s/%s", async (code, message, detail, category) => {
|
|
const f = await fixture(); f.testEnvironment.mockResolvedValue({ ...result(), checks: [{ code, level: "error", message, detail, hint: "private-hint-canary" }] });
|
|
await f.adapter.testEnvironment(context);
|
|
expect((await f.receipts())[0]!.checks).toEqual([{ code, level: "error", category }]);
|
|
expect(JSON.stringify(await f.receipts())).not.toMatch(/private-|APIError|statusCode|detail|message|hint/);
|
|
});
|
|
it.each([
|
|
["HTTP 401 private", "authentication"], ["ECONNREFUSED private", "network"], ["EPERM private", "permission"],
|
|
["ENOENT private", "command-missing"], ["timed out private", "timeout"], ["unrecognized private", "unknown"],
|
|
])("classifies only a closed category for %s", (value, category) => expect(readinessErrorCategory(value)).toBe(category));
|
|
});
|