mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-13 04:02:28 +02:00
DottaandPaperclip
638296ab2a
Cover native edit approval within the authorized invocation and record bounded, versioned tool grants per conversation. Recheck policy and cancellation before each operation. Restore named custom connections through their configured identity and reject native route fallback. Bind assigned skill contents across temporary lease replacements while preserving their write protection. Qualify real native once/deny/session file effects, provider restart, distinct same-name tool identities, conversation isolation and unanswered permission cancellation with the credential-free production ACPX fixture. Update the reviewed platform closure pins and document the remaining release gates. Co-Authored-By: Paperclip <noreply@paperclip.ing>
482 lines
34 KiB
JavaScript
482 lines
34 KiB
JavaScript
// Pinned native Hermes + production ACPX host, deterministic model fixture.
|
|
// This is transport proof, not live model or product qualification.
|
|
// PAPERCLIP_HERMES_QUALIFY=1 node --test scripts/qualify-hermes-runtime.test.mjs
|
|
import assert from "node:assert/strict";
|
|
import { execFile } from "node:child_process";
|
|
import { once } from "node:events";
|
|
import { mkdir, mkdtemp, rm, readFile, readdir, writeFile } from "node:fs/promises";
|
|
import { createServer } from "node:http";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { promisify } from "node:util";
|
|
import test from "node:test";
|
|
import { AcpxRuntimeHost } from "../dist/drivers/acpx/runtime-host.js";
|
|
import { openCodexAcpxRuntime } from "../dist/drivers/acpx/codex-runtime-adapter.js";
|
|
import { acpxProfileClientCapabilities } from "../dist/drivers/acpx/profile-extensions.js";
|
|
import { acpxProviderSessionIdentity } from "../dist/drivers/acpx/recovery-identity.js";
|
|
import { HERMES_CLOSURES } from "../dist/drivers/acpx/hermes-distributions.js";
|
|
import { verifyHermesRuntimeFiles } from "../dist/drivers/acpx/hermes-setup-integrity.js";
|
|
|
|
test("pinned Python qualifies Hermes bridge and wire billing without credentials", {
|
|
skip: process.env.PAPERCLIP_HERMES_QUALIFY !== "1", timeout: 60_000,
|
|
}, async t => {
|
|
const packageRoot = fileURLToPath(new URL("../", import.meta.url));
|
|
const platform = `${process.platform}-${process.arch}`;
|
|
const runtime = join(packageRoot, "provider-assets/hermes", platform);
|
|
assert.ok(HERMES_CLOSURES[platform], "Unqualified Hermes execution platform");
|
|
await verifyHermesRuntimeFiles(runtime, HERMES_CLOSURES[platform]);
|
|
const home = await mkdtemp(join(tmpdir(), "paperclip-hermes-unit-"));
|
|
t.after(() => rm(home, { recursive: true, force: true }));
|
|
const code = `import sys, unittest\nsys.path[:0] = ${JSON.stringify([join(packageRoot, "src/providers/hermes"), join(runtime, "app")])}\n` +
|
|
"suite = unittest.defaultTestLoader.loadTestsFromNames(['test_bridge', 'test_billing'])\n" +
|
|
"sys.exit(0 if unittest.TextTestRunner().run(suite).wasSuccessful() else 1)\n";
|
|
const output = await promisify(execFile)(join(runtime, "python/bin/python3.12"), ["-I", "-B", "-c", code], {
|
|
timeout: 30_000, maxBuffer: 64 * 1024,
|
|
env: { PATH: process.env.PATH, HOME: home, HERMES_HOME: join(home, "hermes") },
|
|
});
|
|
assert.match(output.stderr, /Ran \d+ tests[\s\S]*\bOK\b/);
|
|
t.diagnostic(output.stderr.trim());
|
|
});
|
|
|
|
test("native Hermes permissions cover edits once, survive process restore, and stop unanswered writes", {
|
|
skip: process.env.PAPERCLIP_HERMES_QUALIFY !== "1", timeout: 180_000,
|
|
}, async t => {
|
|
const root = await mkdtemp(join(tmpdir(), "paperclip-hermes-permissions-"));
|
|
const workspace = join(root, "workspace");
|
|
const agentFiles = join(root, "agent-files");
|
|
const skillSource = join(root, "assigned-source");
|
|
const assignedText = "---\nname: assigned\ndescription: Native permission fixture\n---\nRead only assigned instructions.\n";
|
|
await mkdir(workspace); await mkdir(agentFiles); await mkdir(skillSource); await mkdir(join(root, "runtime"));
|
|
await writeFile(join(skillSource, "SKILL.md"), assignedText);
|
|
let host;
|
|
const requests = [];
|
|
const decisions = [];
|
|
const failures = [];
|
|
let stage = "opening";
|
|
const scopeInputs = [];
|
|
const permissionRecords = [];
|
|
let qualified = false;
|
|
const markers = {
|
|
ONCE: [{ path: "once.txt", content: "allowed-once" }],
|
|
DENY: [{ path: "denied.txt", content: "MUST_NOT_EXIST" }],
|
|
GRANT: [{ path: "granted.txt", content: "session-granted" }],
|
|
REUSE: [{ path: "reuse-a.txt", content: "session-reuse-a" }, { path: "reuse-b.txt", content: "session-reuse-b" }],
|
|
PROTECT: [{ path: "set-from-current-assigned-lease", content: "MUST_NOT_REPLACE_ASSIGNED_SKILL" }],
|
|
ISOLATE: [{ path: "isolated.txt", content: "MUST_NOT_EXIST" }],
|
|
STOP: [{ path: "stopped.txt", content: "MUST_NOT_EXIST" }],
|
|
};
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const chunks = [];
|
|
for await (const chunk of req) chunks.push(chunk);
|
|
const body = JSON.parse(Buffer.concat(chunks).toString() || "{}");
|
|
requests.push({ path: req.url, authorizationPresent: req.headers.authorization !== undefined, body });
|
|
if (req.url !== "/v1/chat/completions") { res.writeHead(404).end(); return; }
|
|
assert.equal(body.stream, true, "An undeclared auxiliary model request is not permission proof");
|
|
assert.equal(body.model, "hermes-permission-fixture");
|
|
const index = body.messages.findLastIndex(message => message.role === "user");
|
|
const text = JSON.stringify(body.messages[index].content);
|
|
const marker = Object.keys(markers).find(name => text.includes(`PERMISSION_${name}`));
|
|
assert.ok(marker, "Model request has no exact permission fixture marker");
|
|
const prior = requests.filter(row => row.body.stream && JSON.stringify(row.body.messages[row.body.messages.findLastIndex(message => message.role === "user")]?.content).includes(`PERMISSION_${marker}`));
|
|
assert.ok(prior.length <= 2, "Native fixture attempted an undeclared model retry");
|
|
const results = body.messages.slice(index + 1).filter(message => message.role === "tool");
|
|
res.writeHead(200, { "content-type": "text/event-stream" });
|
|
if (!results.length) {
|
|
if (marker === "PROTECT") markers.PROTECT[0].path = join(scopeInputs.at(-1).assignedSkills[0], "assigned/SKILL.md");
|
|
const tool = body.tools.find(tool => tool.function.name === "write_file");
|
|
assert.ok(tool, "The actual native file tool is missing");
|
|
const calls = markers[marker].map((args, i) => ({ index: i, id: `permission-${marker}-${i}`, type: "function",
|
|
function: { name: tool.function.name, arguments: JSON.stringify(args) } }));
|
|
res.write(`data: ${JSON.stringify({ id: "permission-fixture", choices: [{ index: 0, delta: { role: "assistant", tool_calls: calls }, finish_reason: "tool_calls" }] })}\n\n`);
|
|
} else {
|
|
assert.equal(results.length, markers[marker].length);
|
|
for (const [i, args] of markers[marker].entries()) {
|
|
assert.equal(results[i].tool_call_id, `permission-${marker}-${i}`);
|
|
const content = String(results[i].content);
|
|
if (marker === "PROTECT") {
|
|
assert.match(content, /Paperclip protects/);
|
|
assert.equal(await readFile(args.path, "utf8"), assignedText);
|
|
continue;
|
|
}
|
|
if (marker === "DENY" || marker === "ISOLATE") {
|
|
assert.match(content, /Paperclip denied/);
|
|
assert.equal(await readFile(join(workspace, args.path)).catch(() => null), null);
|
|
} else {
|
|
assert.doesNotMatch(content, /"error"\s*:/);
|
|
assert.equal(await readFile(join(workspace, args.path), "utf8"), args.content);
|
|
}
|
|
}
|
|
res.write(`data: ${JSON.stringify({ id: "permission-fixture", choices: [{ index: 0, delta: { content: `PERMISSION_${marker}_COMPLETE` }, finish_reason: "stop" }] })}\n\n`);
|
|
}
|
|
res.end('data: {"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}\n\ndata: [DONE]\n\n');
|
|
} catch (error) {
|
|
failures.push(String(error));
|
|
// A fixture assertion must settle the native turn rather than trigger
|
|
// SDK retries or conceal its cause behind the outer test timeout.
|
|
if (!res.headersSent) res.writeHead(200, { "content-type": "text/event-stream" });
|
|
res.end('data: {"choices":[{"index":0,"delta":{"content":"PERMISSION_FIXTURE_REJECTED"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n');
|
|
}
|
|
});
|
|
t.after(async () => {
|
|
t.diagnostic(JSON.stringify(qualified ? { qualified, permissionRequests: decisions.length,
|
|
nativeModelCalls: requests.filter(row => row.body.stream).length, restoredPermissionRecords: permissionRecords.length }
|
|
: { stage, failures, scopeInputs, permissionRecords, decisions: decisions.map(row => ({ marker: row.marker,
|
|
title: row.request.toolCall?.title, outcome: row.outcome })), modelCalls: requests.filter(row => row.body.stream).length }));
|
|
await host?.close({ reason: "native permission fixture cleanup" });
|
|
server.closeAllConnections(); await new Promise(resolve => server.close(resolve));
|
|
await rm(root, { recursive: true, force: true });
|
|
});
|
|
server.listen(0, "127.0.0.1"); await once(server, "listening");
|
|
const endpoint = `http://127.0.0.1:${server.address().port}/v1`;
|
|
const options = {
|
|
runtimeDirectory: join(root, "runtime"), normalizedSessionId: "hermes-native-permission-conversation",
|
|
workingDirectory: workspace, agent: "hermes", model: "hermes-permission-fixture", permissionMode: "approve-reads",
|
|
providerPolicy: { readOnly: false }, clientCapabilities: acpxProfileClientCapabilities("hermes"),
|
|
systemInstructions: "Exercise only the supplied native permission fixture.",
|
|
runtimeContext: { instructions: { workingCopy: { kind: "agent_files", rootPath: agentFiles, entryPath: "AGENTS.md" } },
|
|
skills: [{ key: "assigned", runtimeName: "assigned", versionId: "fixture-v1", bundle: {
|
|
schema: "paperclip.runtime-asset.v1", rootPath: skillSource, digest: "3".repeat(64), manifestDigest: "3".repeat(64),
|
|
fileCount: 1, totalBytes: Buffer.byteLength(assignedText),
|
|
} }], mcp: {} },
|
|
environment: { PATH: process.env.PATH, PAPERCLIP_HERMES_CONNECTION_FINGERPRINT: "2".repeat(64),
|
|
PAPERCLIP_HERMES_CONFIG_JSON: JSON.stringify({ model: { provider: "custom:paperclip", default: "hermes-permission-fixture" },
|
|
providers: { paperclip: { base_url: endpoint, transport: "chat_completions", default_model: "hermes-permission-fixture", api_key: "no-key-required" } },
|
|
paperclip_auth: { protocol: "chat", style: "none" } }),
|
|
}, signal: AbortSignal.timeout(150_000),
|
|
};
|
|
const dependencies = { openRuntime: input => {
|
|
scopeInputs.push({ policy: JSON.parse(input.launchEnvironment.PAPERCLIP_HERMES_POLICY),
|
|
assignedSkills: JSON.parse(input.launchEnvironment.PAPERCLIP_HERMES_ASSIGNED_SKILLS) });
|
|
return openCodexAcpxRuntime(input);
|
|
} };
|
|
async function permissionRecord(label) {
|
|
const runtime = fileURLToPath(new URL(`../provider-assets/hermes/${process.platform}-${process.arch}/`, import.meta.url));
|
|
const code = "import json, sqlite3, sys\n" +
|
|
"db=sqlite3.connect('file:'+sys.argv[1]+'?mode=ro',uri=True)\n" +
|
|
"row=db.execute('select model,model_config from sessions where id=?',(sys.argv[2],)).fetchone()\n" +
|
|
"print(json.dumps({'model':row[0],'meta':json.loads(row[1])}))\n";
|
|
const output = await promisify(execFile)(join(runtime, "python/bin/python3.12"), ["-I", "-B", "-c", code,
|
|
join(host.runtimeRoot(), "hermes-home/state.db"), host.identity().backendSessionId], {
|
|
timeout: 10_000, maxBuffer: 64 * 1024, env: { PATH: process.env.PATH, HOME: root },
|
|
});
|
|
const record = JSON.parse(output.stdout);
|
|
permissionRecords.push({ label, ...record });
|
|
return record;
|
|
}
|
|
host = await AcpxRuntimeHost.open(options, dependencies);
|
|
async function run(marker, outcome, expectedDecisions) {
|
|
stage = marker;
|
|
const before = decisions.length;
|
|
const events = [];
|
|
const turn = host.startTurn({ text: `PERMISSION_${marker}`, requestId: `permission-${marker.toLowerCase()}`,
|
|
onPermissionRequest: async request => {
|
|
decisions.push({ marker, request: request.raw, outcome });
|
|
try {
|
|
assert.ok(decisions.length - before <= expectedDecisions, "One operation asked for a second permission or lost its session grant");
|
|
assert.match(request.raw.toolCall.title, /write_file/);
|
|
if (outcome === "allow_always") assert.ok(request.raw.options.some(option => option.optionId === "allow_session" && option.kind === "allow_always"));
|
|
return { outcome };
|
|
} catch (error) {
|
|
failures.push(`${marker}: ${error}`);
|
|
return { outcome: "reject_once" };
|
|
}
|
|
},
|
|
});
|
|
for await (const event of turn.events) events.push(event);
|
|
assert.equal((await turn.result).status, "completed");
|
|
assert.deepEqual(failures, []);
|
|
assert.equal(decisions.length - before, expectedDecisions);
|
|
// Native tool boundaries emit blank message separators. Preserve the
|
|
// complete text while comparing its one scripted answer without that space.
|
|
assert.equal(events.filter(event => event.type === "text_delta" && event.stream !== "thought").map(event => event.text).join("").trim(), `PERMISSION_${marker}_COMPLETE`);
|
|
assert.deepEqual(failures, []);
|
|
return events;
|
|
}
|
|
await run("ONCE", "allow_once", 1);
|
|
await run("DENY", "reject_once", 1);
|
|
await run("GRANT", "allow_always", 1);
|
|
const granted = await permissionRecord("granted");
|
|
assert.deepEqual(granted.meta.paperclip_permissions.tools, ["write_file"]);
|
|
const identity = acpxProviderSessionIdentity(host.identity(), host.binding());
|
|
stage = "restoring";
|
|
await host.close({ reason: "permission per-turn process restoration" });
|
|
assert.deepEqual(await permissionRecord("provider-stopped"), granted);
|
|
host = await AcpxRuntimeHost.open({ ...options, expectedIdentity: identity }, dependencies);
|
|
assert.deepEqual(await permissionRecord("provider-restored"), granted);
|
|
const reused = await run("REUSE", "reject_once", 0);
|
|
assert.notDeepEqual(scopeInputs[0].assignedSkills, scopeInputs[1].assignedSkills, "Restore did not replace the disposable assigned-skill lease");
|
|
const ids = markers.REUSE.map((_args, i) => `permission-REUSE-${i}`);
|
|
for (const id of ids) {
|
|
assert.equal(reused.filter(event => event.type === "tool_call" && event.tag === "tool_call" && event.toolCallId === id).length, 1);
|
|
assert.equal(reused.filter(event => event.type === "tool_call" && event.tag === "tool_call_update" && event.status === "completed" && event.toolCallId === id).length, 1);
|
|
}
|
|
await run("PROTECT", "reject_once", 0);
|
|
await host.close({ reason: "permission conversation isolation" });
|
|
host = await AcpxRuntimeHost.open({ ...options, normalizedSessionId: "hermes-native-permission-other-conversation" }, dependencies);
|
|
await run("ISOLATE", "reject_once", 1);
|
|
stage = "STOP";
|
|
let entered;
|
|
const waiting = new Promise(resolve => { entered = resolve; });
|
|
const stop = host.startTurn({ text: "PERMISSION_STOP", requestId: "permission-stop",
|
|
onPermissionRequest: async (request, context) => {
|
|
assert.match(request.raw.toolCall.title, /stopped.txt/);
|
|
decisions.push({ marker: "STOP", request: request.raw, outcome: "cancel" });
|
|
entered();
|
|
return await new Promise(resolve => {
|
|
if (context.signal.aborted) resolve({ outcome: "cancel" });
|
|
else context.signal.addEventListener("abort", () => resolve({ outcome: "cancel" }), { once: true });
|
|
});
|
|
},
|
|
});
|
|
const drained = (async () => { for await (const _event of stop.events) { /* drain */ } })();
|
|
await Promise.race([waiting, stop.result.then(result => { throw new Error(`Stop fixture ended without its permission request: ${JSON.stringify(result)}`); })]);
|
|
await host.interruptActiveTurn("stop unanswered native permission");
|
|
await drained;
|
|
assert.equal((await stop.result).status, "cancelled");
|
|
assert.equal(await readFile(join(workspace, "stopped.txt")).catch(() => null), null);
|
|
assert.ok(requests.every(row => !row.authorizationPresent));
|
|
assert.deepEqual(failures, []);
|
|
assert.deepEqual(Object.fromEntries(Object.keys(markers).map(marker => [marker, requests.filter(row => row.body.stream &&
|
|
JSON.stringify(row.body.messages[row.body.messages.findLastIndex(message => message.role === "user")]?.content).includes(`PERMISSION_${marker}`)).length])),
|
|
{ ONCE: 2, DENY: 2, GRANT: 2, REUSE: 2, PROTECT: 2, ISOLATE: 2, STOP: 1 });
|
|
qualified = true;
|
|
t.diagnostic("Native once/deny/session, provider process restoration with a new assigned-skill lease, protected skill writes, two same-name tool identities, conversation isolation and unanswered permission Stop passed. No paid model or browser proof.");
|
|
});
|
|
|
|
test("pinned Hermes streams through the production ACPX host using a no-auth local connection", {
|
|
skip: process.env.PAPERCLIP_HERMES_QUALIFY !== "1", timeout: 180_000,
|
|
}, async (t) => {
|
|
const root = await mkdtemp(join(tmpdir(), "paperclip-hermes-transport-"));
|
|
const workspace = join(root, "workspace");
|
|
const agentFiles = join(root, "agent-files");
|
|
await mkdir(workspace); await mkdir(agentFiles); await mkdir(join(root, "runtime"));
|
|
let host;
|
|
const requests = [];
|
|
const bridgeCalls = [];
|
|
const server = createServer(async (req, res) => {
|
|
const chunks = [];
|
|
for await (const chunk of req) chunks.push(chunk);
|
|
const body = JSON.parse(Buffer.concat(chunks).toString() || "{}");
|
|
requests.push({ path: req.url, authorization: req.headers.authorization, body });
|
|
if (req.url !== "/v1/chat/completions") { res.writeHead(404).end(); return; }
|
|
if (!body.stream) {
|
|
res.writeHead(200, { "content-type": "application/json" });
|
|
res.end(JSON.stringify({ id: "fixture", object: "chat.completion", model: "hermes-fixture", choices: [{ index: 0, message: { role: "assistant", content: "Fixture title" }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 } }));
|
|
return;
|
|
}
|
|
res.writeHead(200, { "content-type": "text/event-stream" });
|
|
const lastUser = body.messages.findLastIndex(message => message.role === "user");
|
|
const userText = JSON.stringify(body.messages[lastUser]?.content);
|
|
if ((userText.includes("NATIVE_STEER_FIXTURE") && !JSON.stringify(body.messages).includes("NATIVE_STEER_ACCEPTED")) || userText.includes("NATIVE_STOP_FIXTURE")) {
|
|
for (let index = 0; index < 150 && !res.destroyed; index++) {
|
|
res.write(`data: ${JSON.stringify({ id: "control-fixture", choices: [{ index: 0, delta: { reasoning_content: "Working. " }, finish_reason: null }] })}\n\n`);
|
|
await new Promise(resolve => setTimeout(resolve, 30));
|
|
}
|
|
if (!res.destroyed) res.end('data: {"id":"control-fixture","choices":[{"index":0,"delta":{"content":"Control fixture complete."},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n');
|
|
return;
|
|
}
|
|
const toolDone = body.messages.slice(lastUser + 1).some(message => message.role === "tool");
|
|
if (!toolDone && (userText.includes("NATIVE_TOOL_FIXTURE") || userText.includes("NATIVE_QUESTION_FIXTURE") || userText.includes("NATIVE_MCP_FIXTURE") || userText.includes("NATIVE_MEMORY_FIXTURE") || userText.includes("NATIVE_PLAN_FIXTURE") || userText.includes("NATIVE_GOVERNED_QUESTION_FIXTURE"))) {
|
|
const question = userText.includes("NATIVE_QUESTION_FIXTURE");
|
|
const governed = userText.includes("NATIVE_GOVERNED_QUESTION_FIXTURE");
|
|
const bridge = userText.includes("NATIVE_MCP_FIXTURE");
|
|
const memory = userText.includes("NATIVE_MEMORY_FIXTURE");
|
|
const plan = userText.includes("NATIVE_PLAN_FIXTURE");
|
|
const name = governed ? body.tools.find(tool => tool.function?.name.endsWith("request_human_input"))?.function.name ?? "missing_question_tool"
|
|
: question ? "clarify" : memory ? "memory" : bridge ? body.tools.find(tool => tool.function?.name.endsWith("fixture_probe"))?.function.name ?? "missing_assigned_tool"
|
|
: plan ? body.tools.find(tool => tool.function?.name.endsWith("write_document"))?.function.name ?? "missing_plan_tool" : "terminal";
|
|
const args = question ? { question: "Choose the fixture color", choices: ["Cobalt", "Amber"] }
|
|
: memory ? { action: "add", target: "memory", content: "NATIVE_PERSISTED_MEMORY: Project uses Cobalt." }
|
|
: governed || bridge ? { marker: "assigned-tool-roundtrip" }
|
|
: plan ? { marker: "planning-workflow-roundtrip" }
|
|
: { command: "printf 'native-hermes-command\\n' > command-proof.txt", timeout: 10 };
|
|
res.write(`data: ${JSON.stringify({ id: "tool-fixture", object: "chat.completion.chunk", model: "hermes-fixture", created: 1, choices: [{ index: 0, delta: { role: "assistant", tool_calls: [{ index: 0, id: `native-${name}`, type: "function", function: { name, arguments: JSON.stringify(args) } }] }, finish_reason: null }] })}\n\n`);
|
|
res.end(`data: ${JSON.stringify({ id: "tool-fixture", choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } })}\n\ndata: [DONE]\n\n`);
|
|
return;
|
|
}
|
|
for (const [delta, finish_reason] of [
|
|
[{ role: "assistant", reasoning_content: "Fixture reasoning." }, null],
|
|
[{ content: "Hermes " }, null], [{ content: "transport works." }, null], [{}, "stop"],
|
|
]) {
|
|
res.write(`data: ${JSON.stringify({ id: "fixture", object: "chat.completion.chunk", model: "hermes-fixture", created: 1, choices: [{ index: 0, delta, finish_reason }] })}\n\n`);
|
|
await new Promise(resolve => setTimeout(resolve, 40));
|
|
}
|
|
res.end('data: {"id":"fixture","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}\n\ndata: [DONE]\n\n');
|
|
});
|
|
t.after(async () => {
|
|
if (host) await host.close({ reason: "qualification cleanup" });
|
|
server.closeAllConnections(); await new Promise(resolve => server.close(resolve));
|
|
if (process.env.PAPERCLIP_HERMES_QUALIFY_KEEP === "1") t.diagnostic(`Private fixture evidence: ${root}`);
|
|
else await rm(root, { recursive: true, force: true });
|
|
});
|
|
server.listen(0, "127.0.0.1"); await once(server, "listening");
|
|
const endpoint = `http://127.0.0.1:${server.address().port}/v1`;
|
|
const options = {
|
|
runtimeDirectory: join(root, "runtime"), normalizedSessionId: "hermes-transport-test", workingDirectory: workspace,
|
|
agent: "hermes", model: "hermes-fixture", permissionMode: "approve-all", providerPolicy: { readOnly: false },
|
|
clientCapabilities: acpxProfileClientCapabilities("hermes"),
|
|
systemInstructions: "This is a transport fixture. Reply briefly.",
|
|
semanticTools: {
|
|
tools: ["fixture_probe", "write_document", "request_human_input"].map(name => ({ name, description: "Return the supplied marker (simulated semantic authority).", inputSchema: { type: "object", properties: { marker: { type: "string" } }, required: ["marker"], additionalProperties: false } })),
|
|
handler: async call => {
|
|
bridgeCalls.push({ tool: call.tool, arguments: call.arguments });
|
|
return call.tool === "request_human_input" ? { disposition: "applied", interaction: {
|
|
id: "fixture-question", companyId: "fixture-company", issueId: "fixture-issue", sourceRunId: "fixture-run",
|
|
kind: "ask_user_questions", status: "pending", continuationPolicy: "wake_assignee",
|
|
} } : { marker: "assigned-tool-returned" };
|
|
},
|
|
},
|
|
runtimeContext: { instructions: { workingCopy: { kind: "agent_files", rootPath: agentFiles, entryPath: "AGENTS.md" } }, skills: [], mcp: {} },
|
|
environment: { PATH: process.env.PATH,
|
|
PAPERCLIP_HERMES_CONNECTION_FINGERPRINT: "1".repeat(64),
|
|
PAPERCLIP_HERMES_CONFIG_JSON: JSON.stringify({ model: { provider: "custom:paperclip", default: "hermes-fixture" },
|
|
providers: { paperclip: { base_url: endpoint, transport: "chat_completions", default_model: "hermes-fixture", api_key: "no-key-required" } },
|
|
paperclip_auth: { protocol: "chat", style: "none" } }),
|
|
}, signal: AbortSignal.timeout(150_000),
|
|
};
|
|
const dependencies = { openRuntime: openCodexAcpxRuntime, reportRetainedCleanupFailure: failure => t.diagnostic(String(failure.error)) };
|
|
host = await AcpxRuntimeHost.open(options, dependencies);
|
|
const turn = host.startTurn({ text: "Reply with the transport marker.", requestId: "transport-turn" });
|
|
const events = [];
|
|
let completed = false;
|
|
turn.result.then(() => { completed = true; });
|
|
for await (const event of turn.events) events.push({ event, beforeCompletion: !completed });
|
|
assert.equal((await turn.result).status, "completed");
|
|
const text = events.filter(({ event }) => event.type === "text_delta" && event.stream !== "thought");
|
|
assert.equal(text.map(({ event }) => event.text).join(""), "Hermes transport works.");
|
|
assert.ok(text.some(row => row.beforeCompletion));
|
|
assert.ok(requests.some(row => row.body.model === "hermes-fixture"));
|
|
assert.ok(requests.every(row => row.authorization === undefined), `No-auth connection sent credentials: ${JSON.stringify(requests.map(row => ({ path: row.path, authenticated: row.authorization !== undefined })))}`);
|
|
assert.ok(events.some(({ event }) => event.type === "text_delta" && event.stream === "thought"));
|
|
const expectedIdentity = acpxProviderSessionIdentity(host.identity(), host.binding());
|
|
await host.close({ reason: "per-turn process checkpoint" });
|
|
host = await AcpxRuntimeHost.open({ ...options, expectedIdentity }, dependencies);
|
|
const imageData = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aA1sAAAAASUVORK5CYII=";
|
|
const resumed = host.startTurn({ text: "Continue the previous conversation and inspect this image.", requestId: "resumed-turn",
|
|
attachments: [{ schema: "paperclip.user_attachment.v1", kind: "image", name: "pixel.png", mediaType: "image/png", data: imageData }] });
|
|
const resumedEvents = [];
|
|
for await (const event of resumed.events) resumedEvents.push(event);
|
|
assert.equal(resumedEvents.filter(event => event.type === "text_delta" && event.stream !== "thought").map(event => event.text).join(""), "Hermes transport works.", "Restored history was emitted as new assistant output");
|
|
assert.equal((await resumed.result).status, "completed");
|
|
assert.ok(requests.some(row => row.body.messages?.filter(message => message.role === "user").length >= 2), "Restore lost the previous conversation");
|
|
assert.ok(requests.some(row => row.body.messages?.some(message => Array.isArray(message.content) && message.content.some(part => part.type === "image_url" && part.image_url.url === `data:image/png;base64,${imageData}`))), `The selected model did not receive the authorized image bytes: ${JSON.stringify(requests.filter(row => row.body.stream).map(row => row.body.messages.filter(message => message.role === "user")))}`);
|
|
const tools = host.startTurn({ text: "NATIVE_TOOL_FIXTURE: Run the supplied command.", requestId: "tools-turn" });
|
|
const toolEvents = [];
|
|
for await (const event of tools.events) toolEvents.push(event);
|
|
assert.equal((await tools.result).status, "completed");
|
|
assert.equal(await readFile(join(workspace, "command-proof.txt"), "utf8").catch(() => null), "native-hermes-command\n",
|
|
`Native command failed: ${JSON.stringify(requests.flatMap(row => row.body.messages?.filter(message => message.role === "tool") ?? []))}`);
|
|
assert.ok(toolEvents.some(event => event.type === "tool_call" || event.type === "tool_call_update"), "Native tool events did not reach ACPX");
|
|
let questions = 0;
|
|
const questionTurn = host.startTurn({ text: "NATIVE_QUESTION_FIXTURE: Ask the native question.", requestId: "question-turn",
|
|
onExtensionRequest: async (method, params) => {
|
|
assert.equal(method, "_hermes/ask_questions");
|
|
assert.equal(params.version, 1);
|
|
questions++;
|
|
const question = params.input.questions[0];
|
|
assert.ok(question.options[0].label.startsWith("Cobalt"));
|
|
return { outcome: "answered", answers: { [question.id]: { selectedOptionIds: ["o0"] } } };
|
|
},
|
|
});
|
|
for await (const _event of questionTurn.events) { /* drain */ }
|
|
assert.equal((await questionTurn.result).status, "completed");
|
|
assert.equal(questions, 1, `Native clarify failed: ${JSON.stringify(requests.at(-1)?.body.messages?.filter(message => message.role === "tool"))}`);
|
|
assert.ok(requests.some(row => row.body.messages?.some(message => message.role === "tool" && String(message.content).includes("Cobalt"))), "The native callback did not receive the submitted answer");
|
|
const governedRequestsBefore = requests.length;
|
|
const governedOrder = [];
|
|
const governed = host.startTurn({ text: "NATIVE_GOVERNED_QUESTION_FIXTURE: Save the assigned Paperclip question.", requestId: "governed-question-turn",
|
|
onExtensionNotification: async (method, params) => {
|
|
if (method === "_hermes/usage") {
|
|
assert.equal(params.tokens, "reported");
|
|
governedOrder.push("usage");
|
|
}
|
|
},
|
|
});
|
|
const governedEvents = [];
|
|
for await (const event of governed.events) {
|
|
governedEvents.push(event);
|
|
if (event.type === "tool_call" && event.tag === "tool_call_update" && event.status === "completed") governedOrder.push("question");
|
|
}
|
|
assert.equal((await governed.result).status, "cancelled", "Hermes continued after the committed question");
|
|
assert.equal(requests.length - governedRequestsBefore, 1, "Hermes started another model request after saving the question");
|
|
assert.deepEqual(governedOrder, ["usage", "question"], "The committed question result preceded final native usage");
|
|
assert.ok(governedEvents.some(event => event.type === "tool_call" && event.tag === "tool_call_update"
|
|
&& event.status === "completed" && JSON.stringify(event.rawOutput).includes("fixture-question")),
|
|
`The committed native tool result was lost at interruption: ${JSON.stringify(governedEvents).slice(0, 8000)}`);
|
|
for (const kind of ["MCP", "MEMORY"]) {
|
|
const turn = host.startTurn({ text: `NATIVE_${kind}_FIXTURE: Use the native tool.`, requestId: `${kind.toLowerCase()}-turn` });
|
|
for await (const _event of turn.events) { /* drain */ }
|
|
assert.equal((await turn.result).status, "completed");
|
|
}
|
|
assert.deepEqual(bridgeCalls, ["request_human_input", "fixture_probe"].map(tool => ({ tool, arguments: { marker: "assigned-tool-roundtrip" } })));
|
|
assert.equal(host.steeringCapability()?.steering, true);
|
|
assert.equal(host.steeringCapability()?.queuedFollowUp, false);
|
|
const steering = host.startTurn({ text: "NATIVE_STEER_FIXTURE: Continue working until redirected.", requestId: "steering-turn" });
|
|
let steered = false;
|
|
for await (const event of steering.events) {
|
|
if (!steered && event.type === "text_delta") {
|
|
await host.steerActiveTurn("NATIVE_STEER_ACCEPTED: Answer with the transport marker instead.");
|
|
steered = true;
|
|
}
|
|
}
|
|
assert.equal(steered, true);
|
|
assert.equal((await steering.result).status, "completed");
|
|
assert.ok(requests.some(row => JSON.stringify(row.body.messages ?? []).includes("NATIVE_STEER_ACCEPTED")), "Accepted steering did not reach the native turn");
|
|
await assert.rejects(host.steerActiveTurn("stale correction"), /active turn/);
|
|
const stop = host.startTurn({ text: "NATIVE_STOP_FIXTURE: Continue working until cancelled.", requestId: "stop-turn" });
|
|
let stopped = false;
|
|
for await (const event of stop.events) {
|
|
if (!stopped && event.type === "text_delta") {
|
|
stopped = true;
|
|
await host.interruptActiveTurn("qualification stop");
|
|
}
|
|
}
|
|
assert.equal(stopped, true);
|
|
assert.equal((await stop.result).status, "cancelled");
|
|
const stoppedIdentity = acpxProviderSessionIdentity(host.identity(), host.binding());
|
|
await host.close({ reason: "restore cancelled turn" });
|
|
host = await AcpxRuntimeHost.open({ ...options, expectedIdentity: stoppedIdentity }, dependencies);
|
|
const afterStop = host.startTurn({ text: "Reply with a fresh transport marker after cancellation.", requestId: "after-stop-turn" });
|
|
const afterStopEvents = [];
|
|
for await (const event of afterStop.events) afterStopEvents.push(event);
|
|
const afterStopResult = await afterStop.result;
|
|
assert.equal(afterStopResult.status, "completed", JSON.stringify(afterStopResult));
|
|
assert.equal(afterStopEvents.filter(event => event.type === "text_delta" && event.stream !== "thought").map(event => event.text).join(""), "Hermes transport works.", "Cancelled history or old tool results polluted the restored turn");
|
|
assert.ok(!afterStopEvents.some(event => event.type === "tool_call" || event.type === "tool_call_update"), "Old tools replayed as active work");
|
|
assert.ok(requests.at(-1).body.messages.some(message => message.role === "user" && JSON.stringify(message.content).includes("NATIVE_STOP_FIXTURE")), "Cancelled turn history was discarded instead of restored");
|
|
const restoreIdentity = acpxProviderSessionIdentity(host.identity(), host.binding());
|
|
await host.close({ reason: "verify missing history" }); host = undefined;
|
|
assert.match(await readFile(join(agentFiles, "hermes/memories/MEMORY.md"), "utf8"), /NATIVE_PERSISTED_MEMORY/);
|
|
const nativeDatabases = (await readdir(join(root, "runtime"), { recursive: true })).filter(path => path.endsWith("state.db"));
|
|
assert.equal(nativeDatabases.length, 1);
|
|
await rm(join(root, "runtime", nativeDatabases[0]));
|
|
let refused = false;
|
|
try {
|
|
host = await AcpxRuntimeHost.open({ ...options, expectedIdentity: restoreIdentity }, dependencies);
|
|
const missing = host.startTurn({ text: "Continue the lost conversation.", requestId: "missing-history" });
|
|
for await (const _event of missing.events) { /* drain */ }
|
|
refused = (await missing.result).status !== "completed";
|
|
} catch (error) {
|
|
assert.match(String(error), /restore|history|session|resume/i);
|
|
refused = true;
|
|
}
|
|
assert.equal(refused, true, "Missing native history was silently replaced with a new conversation");
|
|
await host?.close({ reason: "planning fixture admission" });
|
|
host = await AcpxRuntimeHost.open({ ...options, normalizedSessionId: "hermes-planning-test", providerPolicy: { readOnly: true } }, dependencies);
|
|
const planning = host.startTurn({ text: "NATIVE_PLAN_FIXTURE: Use the assigned plan document tool.", requestId: "planning-turn" });
|
|
const planningEvents = [];
|
|
for await (const event of planning.events) planningEvents.push(event);
|
|
assert.equal((await planning.result).status, "completed");
|
|
assert.deepEqual(bridgeCalls.at(-1), { tool: "write_document", arguments: { marker: "planning-workflow-roundtrip" } });
|
|
assert.ok(planningEvents.some(event => (event.type === "tool_call" || event.type === "tool_call_update") && JSON.stringify(event).includes("assigned-tool-returned")), "Structured native tool output was omitted from the transcript");
|
|
await rm(join(workspace, "command-proof.txt"));
|
|
const forbidden = host.startTurn({ text: "NATIVE_TOOL_FIXTURE: Run the supplied command while planning.", requestId: "planning-forbidden-turn" });
|
|
const forbiddenEvents = [];
|
|
for await (const event of forbidden.events) forbiddenEvents.push(event);
|
|
assert.equal((await forbidden.result).status, "completed");
|
|
assert.equal(await readFile(join(workspace, "command-proof.txt"), "utf8").catch(() => null), null, "Planning mode executed a native write");
|
|
assert.ok(forbiddenEvents.some(event => (event.type === "tool_call" || event.type === "tool_call_update") && JSON.stringify(event).includes("planning mode permits")), `Native policy denial was omitted from the transcript: ${JSON.stringify({ events: forbiddenEvents.filter(event => event.type === "tool_call" || event.type === "tool_call_update"), results: requests.at(-1)?.body.messages.filter(message => message.role === "tool").slice(-1) })}`);
|
|
});
|