From 772db06812d34facfb358c3fb0c9bdffc21aad68 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 7 Oct 2026 06:56:20 -0500 Subject: [PATCH] fix(hermes): keep native candidate independently restorable Reject replaced learned-state parents, include authenticated run attachment and the advertised native transport fixtures in this candidate. Keep paid browser campaign definitions in the qualification PR. Co-Authored-By: Paperclip --- .../runner-core/src/acpx_provider_backend.rs | 8 +- .../src/acpx_provider_capabilities.rs | 14 +- .../scripts/qualify-hermes-runnerd.test.mjs | 123 +++++++++++ .../scripts/qualify-hermes-runtime.test.mjs | 202 ++++++++++++++++++ .../src/drivers/acpx/hermes-state.test.ts | 18 +- .../src/drivers/acpx/hermes-state.ts | 61 +++++- tests/runner-e2e/README.md | 7 +- tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 9 +- tests/runner-e2e/extended-harnesses.test.ts | 5 +- tests/runner-e2e/live-fixtures.test.ts | 3 +- tests/runner-e2e/live-fixtures.ts | 3 +- tests/runner-e2e/types.ts | 2 +- 13 files changed, 423 insertions(+), 38 deletions(-) create mode 100644 packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs create mode 100644 packages/paperclip-runner/scripts/qualify-hermes-runtime.test.mjs diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 175f310582d..b222a0bbb3c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -3266,9 +3266,15 @@ mod tests { authenticated_run_grant_attachment("claude"); } + #[cfg(unix)] + #[test] + fn hermes_attachment_rotates_only_authenticated_run_grants() { + authenticated_run_grant_attachment("hermes"); + } + #[cfg(unix)] fn authenticated_run_grant_attachment(agent: &str) { - let directory = temporary_directory("cursor-cross-run-attach"); + let directory = temporary_directory(&format!("{agent}-cross-run-attach")); let runtime = directory.join("runtime"); let workspace = directory.join("workspace"); fs::create_dir_all(&runtime).unwrap(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_capabilities.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_capabilities.rs index 830e31b1fe3..a6600802101 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_capabilities.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_capabilities.rs @@ -11,7 +11,7 @@ pub(crate) enum RunAttachmentPolicy { pub(crate) fn run_attachment_policy(agent: &str) -> RunAttachmentPolicy { match agent { - "cursor" => RunAttachmentPolicy::AuthenticatedRunGrants, + "cursor" | "hermes" => RunAttachmentPolicy::AuthenticatedRunGrants, "claude" => RunAttachmentPolicy::AuthenticatedAssetPaths, _ => RunAttachmentPolicy::ImmutableInstructions, } @@ -22,11 +22,13 @@ mod tests { use super::*; #[test] - fn only_qualified_providers_opt_into_run_grant_rotation() { - assert_eq!( - run_attachment_policy("cursor"), - RunAttachmentPolicy::AuthenticatedRunGrants - ); + fn only_explicit_profiles_opt_into_run_grant_rotation() { + for agent in ["cursor", "hermes"] { + assert_eq!( + run_attachment_policy(agent), + RunAttachmentPolicy::AuthenticatedRunGrants + ); + } assert_eq!( run_attachment_policy("claude"), RunAttachmentPolicy::AuthenticatedAssetPaths diff --git a/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs b/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs new file mode 100644 index 00000000000..62b2598b48a --- /dev/null +++ b/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs @@ -0,0 +1,123 @@ +// Production TS -> Rust PRP -> sidecar -> ACPX -> pinned Hermes -> deterministic +// HTTP model fixture. This is not live provider or browser qualification. +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { once } from 'node:events'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { createRunnerdNativeSessionBackend } from '../dist/backends/codex-native-backend.js'; +import { createRunnerdCodexTransport } from '../dist/live/runnerd-codex-transport.js'; +import { resolveQualifiedAcpxProfile } from '../dist/drivers/acpx/qualified-profiles.js'; +import { parseNativeExecutionInput } from '../dist/contracts/native-execution.js'; +import { PAPERCLIP_EXECUTION_PROMPT, PAPERCLIP_EXECUTION_PROMPT_REVISION, nativeRuntimePromptDigest, canonicalNativeRuntimeContextDigest } from '../dist/contracts/runtime-context.js'; + +const sha256 = text => createHash('sha256').update(text).digest('hex'); +const imageData = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aA1sAAAAASUVORK5CYII='; +const completion = { + reportedWorkDisposition: 'done', summary: 'Native Rust path completed.', + completionClaim: { contractRevision: '1', objectiveSatisfied: true, + criteria: [{ criterionId: 'objective', status: 'satisfied', evidenceRefs: [] }], remainingWork: [] }, + evidence: [], verification: [{ commandOrCheck: 'native transport fixture', status: 'passed' }], attentionRequests: [], artifacts: [], +}; + +test('pinned Hermes streams images and semantic completion through Rust PRP and the production sidecar', { + skip: process.env.PAPERCLIP_HERMES_QUALIFY !== '1', timeout: 180_000, +}, async t => { + const root = await mkdtemp(join(tmpdir(), 'paperclip-hermes-runnerd-')); + const workspace = join(root, 'workspace'); + const instructions = join(root, 'instructions'); + const agentFiles = join(root, 'agent-files'); + for (const directory of [workspace, instructions, agentFiles, join(root, 'runtime')]) await mkdir(directory, { mode: 0o700 }); + const entry = 'You are a deterministic native transport test agent.'; + await writeFile(join(instructions, 'AGENTS.md'), entry); + const requests = []; + const server = createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const body = JSON.parse(Buffer.concat(chunks).toString() || '{}'); + requests.push(body); + assert.equal(req.headers.authorization, undefined); + if (req.url !== '/v1/chat/completions') { res.writeHead(404).end(); return; } + if (!body.stream) { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ id: 'title', object: 'chat.completion', model: 'hermes-fixture', choices: [{ index: 0, message: { role: 'assistant', content: 'Fixture' }, finish_reason: 'stop' }] })); + return; + } + res.writeHead(200, { 'content-type': 'text/event-stream' }); + const send = (delta, finish_reason = null) => res.write(`data: ${JSON.stringify({ id: 'native-prp', object: 'chat.completion.chunk', choices: [{ index: 0, delta, finish_reason }] })}\n\n`); + if (!body.messages.some(message => message.role === 'tool')) { + send({ role: 'assistant', reasoning_content: 'Checking the native runner path.' }); + await new Promise(resolve => setTimeout(resolve, 80)); + const finish = body.tools.find(tool => tool.function.name.endsWith('paperclip_finish')); + assert.ok(finish, 'Paperclip completion authority was not exposed to the native model'); + send({ tool_calls: [{ index: 0, id: 'native-prp-finish', type: 'function', function: { name: finish.function.name, arguments: JSON.stringify(completion) } }] }); + send({}, 'tool_calls'); + } else { + send({ role: 'assistant', content: 'Native Rust ' }); + await new Promise(resolve => setTimeout(resolve, 80)); + send({ content: 'path completed.' }); + send({}, 'stop'); + } + res.end('data: {"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}\n\ndata: [DONE]\n\n'); + }); + let session, bundle; + t.after(async () => { + await session?.close({ reason: 'fixture cleanup' }); + await bundle?.transport.close(); + server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); + if (process.env.PAPERCLIP_KEEP_HERMES_FIXTURE === '1') t.diagnostic(`Fixture evidence: ${root}`); + else await rm(root, { recursive: true, force: true }); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const context = { + prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() }, + instructions: { entryPath: 'AGENTS.md', bundle: { schema: 'paperclip.runtime-asset.v1', digest: sha256(entry), manifestDigest: sha256(entry), rootPath: instructions, fileCount: 1, totalBytes: Buffer.byteLength(entry) }, + workingCopy: { kind: 'agent_files', rootPath: agentFiles, entryPath: 'AGENTS.md' } }, + skills: [], mcp: { assignmentSetId: 'none', digest: sha256('none'), bindingId: null }, + }; + const runtimeContext = { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) }; + const identity = { runId: 'hermes-prp-run', sessionId: 'hermes-prp-session', companyId: 'hermes-prp-company', issueId: 'hermes-prp-issue', agentId: 'hermes-prp-agent' }; + const resolved = resolveQualifiedAcpxProfile('hermes', 'hermes-fixture'); + const profile = Object.fromEntries(['driverKind', 'protocolVersion', 'acpxVersion', 'agent', 'agentProfileVersion', 'agentServerPackage', 'agentServerVersion', 'agentRuntimePackage', 'agentRuntimeVersion', 'commandDigest'].map(key => [key, resolved[key]])); + const input = parseNativeExecutionInput({ + schema: 'paperclip.native-execution-input.v6', executionMode: 'default', planningContext: null, + binding: { runId: identity.runId, companyId: identity.companyId, issueId: identity.issueId, agentId: identity.agentId, executionWorkspaceId: 'hermes-prp-workspace' }, + session: { normalizedSessionId: identity.sessionId, driverKind: 'acpx_runtime', protocolVersion: 1 }, + task: { identifier: 'HERMES-PRP', title: 'Native transport fixture', description: null, prompt: 'Complete the native transport fixture.', workMode: 'standard' }, + workspace: { cwd: workspace, repoUrl: null, repoRef: null, branchName: null }, + provider: { kind: 'acpx', agent: 'hermes', model: 'hermes-fixture', permissionMode: 'approve-all', profile, connectionFingerprint: '1'.repeat(64) }, + completionContract: { id: 'hermes-prp-contract', sha256: sha256('fixture'), schemaVersion: 'paperclip.completion-contract.v1', contract: { revision: '1', objective: 'Native transport fixture', criteria: [{ id: 'objective', requirement: 'Complete the native transport fixture.' }] } }, + runtimeContext, interactionResponses: [], credentialBindings: [], + attachments: [{ schema: 'paperclip.user_attachment.v1', kind: 'image', name: 'pixel.png', mediaType: 'image/png', data: imageData }], + }); + bundle = createRunnerdCodexTransport({ + provider: 'acpx', acpxAgent: 'hermes', acpxCandidateProfile: 'hermes', acpxPermissionMode: 'approve-all', + stateDirectory: join(root, 'runner'), acpxRuntimeDirectory: join(root, 'runtime'), runtimeContext, + lifecyclePolicy: { mode: 'per_turn', idleTimeoutMs: null }, turnStartTimeoutMs: 90_000, + onDiagnostic: message => t.diagnostic(message), + environment: { PATH: process.env.PATH, PAPERCLIP_PROVIDER_TRACE_PATH: join(root, 'provider-trace.jsonl'), PAPERCLIP_HERMES_CONNECTION_FINGERPRINT: input.provider.connectionFingerprint, + PAPERCLIP_HERMES_CONFIG_JSON: JSON.stringify({ model: { provider: 'custom:paperclip', default: 'hermes-fixture' }, + providers: { paperclip: { base_url: `http://127.0.0.1:${server.address().port}/v1`, transport: 'chat_completions', default_model: 'hermes-fixture', api_key: 'no-key-required' } }, + paperclip_auth: { protocol: 'chat', style: 'none' } }), + }, + prpIdentity: { runnerInstanceId: 'hermes-prp-runner', environmentLeaseId: 'hermes-prp-lease', runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: 'hermes-prp-turn', itemId: 'hermes-prp-item' }, + }); + const backend = createRunnerdNativeSessionBackend(input, { runnerInstanceId: 'hermes-prp-runner', transportFactory: () => bundle.transport }); + session = await backend.openSession({ identity, workingDirectory: workspace }); + await session.startTurn({ message: { role: 'user', text: input.task.prompt, attachments: input.attachments } }); + const events = []; + for await (const event of session.events()) { + events.push(event); + if (['run.terminal', 'turn.completed', 'turn.failed', 'turn.cancelled', 'turn.interrupted'].includes(event.eventType)) break; + } + assert.ok(events.some(event => event.eventType === 'turn.completed'), JSON.stringify(events)); + assert.ok(events.some(event => JSON.stringify(event.payload).includes('Checking the native runner path.')), 'Reasoning did not cross PRP'); + assert.ok(events.some(event => JSON.stringify(event.payload).includes('Native Rust')), 'Text did not cross PRP'); + assert.ok(requests.some(body => body.messages?.some(message => Array.isArray(message.content) && message.content.some(part => part.type === 'image_url' && part.image_url.url === `data:image/png;base64,${imageData}`))), 'Images did not cross TS/Rust/sidecar'); + const snapshot = await session.snapshot(); + assert.equal(snapshot.semanticResult?.reportedWorkDisposition, 'done'); + assert.equal(snapshot.terminal?.runTerminalState, 'succeeded'); + assert.equal(bundle.evidence().acpxAgent, 'hermes'); +}); diff --git a/packages/paperclip-runner/scripts/qualify-hermes-runtime.test.mjs b/packages/paperclip-runner/scripts/qualify-hermes-runtime.test.mjs new file mode 100644 index 00000000000..4e6ee39d5d4 --- /dev/null +++ b/packages/paperclip-runner/scripts/qualify-hermes-runtime.test.mjs @@ -0,0 +1,202 @@ +// Pinned native Hermes + production ACPX host, deterministic model fixture. +// This is transport proof, not live model or product qualification. +// PAPERCLIP_HERMES_QUALIFY=1 node --test scripts/qualify-hermes-runtime.test.mjs +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { mkdir, mkdtemp, rm, readFile, readdir } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { AcpxRuntimeHost } from "../dist/drivers/acpx/runtime-host.js"; +import { openCodexAcpxRuntime } from "../dist/drivers/acpx/codex-runtime-adapter.js"; +import { acpxProfileClientCapabilities } from "../dist/drivers/acpx/profile-extensions.js"; +import { acpxProviderSessionIdentity } from "../dist/drivers/acpx/recovery-identity.js"; + +test("pinned Hermes streams through the production ACPX host using a no-auth local connection", { + skip: process.env.PAPERCLIP_HERMES_QUALIFY !== "1", timeout: 180_000, +}, async (t) => { + const root = await mkdtemp(join(tmpdir(), "paperclip-hermes-transport-")); + const workspace = join(root, "workspace"); + const agentFiles = join(root, "agent-files"); + await mkdir(workspace); await mkdir(agentFiles); await mkdir(join(root, "runtime")); + let host; + const requests = []; + const bridgeCalls = []; + const server = createServer(async (req, res) => { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const body = JSON.parse(Buffer.concat(chunks).toString() || "{}"); + requests.push({ path: req.url, authorization: req.headers.authorization, body }); + if (req.url !== "/v1/chat/completions") { res.writeHead(404).end(); return; } + if (!body.stream) { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ id: "fixture", object: "chat.completion", model: "hermes-fixture", choices: [{ index: 0, message: { role: "assistant", content: "Fixture title" }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 } })); + return; + } + res.writeHead(200, { "content-type": "text/event-stream" }); + const lastUser = body.messages.findLastIndex(message => message.role === "user"); + const userText = JSON.stringify(body.messages[lastUser]?.content); + if ((userText.includes("NATIVE_STEER_FIXTURE") && !JSON.stringify(body.messages).includes("NATIVE_STEER_ACCEPTED")) || userText.includes("NATIVE_STOP_FIXTURE")) { + for (let index = 0; index < 150 && !res.destroyed; index++) { + res.write(`data: ${JSON.stringify({ id: "control-fixture", choices: [{ index: 0, delta: { reasoning_content: "Working. " }, finish_reason: null }] })}\n\n`); + await new Promise(resolve => setTimeout(resolve, 30)); + } + if (!res.destroyed) res.end('data: {"id":"control-fixture","choices":[{"index":0,"delta":{"content":"Control fixture complete."},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n'); + return; + } + const toolDone = body.messages.slice(lastUser + 1).some(message => message.role === "tool"); + if (!toolDone && (userText.includes("NATIVE_TOOL_FIXTURE") || userText.includes("NATIVE_QUESTION_FIXTURE") || userText.includes("NATIVE_MCP_FIXTURE") || userText.includes("NATIVE_MEMORY_FIXTURE"))) { + const question = userText.includes("NATIVE_QUESTION_FIXTURE"); + const bridge = userText.includes("NATIVE_MCP_FIXTURE"); + const memory = userText.includes("NATIVE_MEMORY_FIXTURE"); + const name = question ? "clarify" : memory ? "memory" : bridge ? body.tools.find(tool => tool.function?.name.endsWith("fixture_probe"))?.function.name ?? "missing_assigned_tool" : "terminal"; + const args = question ? { question: "Choose the fixture color", choices: ["Cobalt", "Amber"] } + : memory ? { action: "add", target: "memory", content: "NATIVE_PERSISTED_MEMORY: Project uses Cobalt." } + : bridge ? { marker: "assigned-tool-roundtrip" } + : { command: "printf 'native-hermes-command\\n' > command-proof.txt", timeout: 10 }; + res.write(`data: ${JSON.stringify({ id: "tool-fixture", object: "chat.completion.chunk", model: "hermes-fixture", created: 1, choices: [{ index: 0, delta: { role: "assistant", tool_calls: [{ index: 0, id: `native-${name}`, type: "function", function: { name, arguments: JSON.stringify(args) } }] }, finish_reason: null }] })}\n\n`); + res.end(`data: ${JSON.stringify({ id: "tool-fixture", choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } })}\n\ndata: [DONE]\n\n`); + return; + } + for (const [delta, finish_reason] of [ + [{ role: "assistant", reasoning_content: "Fixture reasoning." }, null], + [{ content: "Hermes " }, null], [{ content: "transport works." }, null], [{}, "stop"], + ]) { + res.write(`data: ${JSON.stringify({ id: "fixture", object: "chat.completion.chunk", model: "hermes-fixture", created: 1, choices: [{ index: 0, delta, finish_reason }] })}\n\n`); + await new Promise(resolve => setTimeout(resolve, 40)); + } + res.end('data: {"id":"fixture","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}\n\ndata: [DONE]\n\n'); + }); + t.after(async () => { + if (host) await host.close({ reason: "qualification cleanup" }); + server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); + if (process.env.PAPERCLIP_HERMES_QUALIFY_KEEP === "1") t.diagnostic(`Private fixture evidence: ${root}`); + else await rm(root, { recursive: true, force: true }); + }); + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const endpoint = `http://127.0.0.1:${server.address().port}/v1`; + const options = { + runtimeDirectory: join(root, "runtime"), normalizedSessionId: "hermes-transport-test", workingDirectory: workspace, + agent: "hermes", model: "hermes-fixture", permissionMode: "approve-all", providerPolicy: { readOnly: false }, + clientCapabilities: acpxProfileClientCapabilities("hermes"), + systemInstructions: "This is a transport fixture. Reply briefly.", + semanticTools: { + tools: [{ name: "fixture_probe", description: "Return the supplied marker.", inputSchema: { type: "object", properties: { marker: { type: "string" } }, required: ["marker"], additionalProperties: false } }], + handler: async call => { bridgeCalls.push({ tool: call.tool, arguments: call.arguments }); return { marker: "assigned-tool-returned" }; }, + }, + runtimeContext: { instructions: { workingCopy: { kind: "agent_files", rootPath: agentFiles, entryPath: "AGENTS.md" } }, skills: [], mcp: {} }, + environment: { PATH: process.env.PATH, + PAPERCLIP_HERMES_CONNECTION_FINGERPRINT: "1".repeat(64), + PAPERCLIP_HERMES_CONFIG_JSON: JSON.stringify({ model: { provider: "custom:paperclip", default: "hermes-fixture" }, + providers: { paperclip: { base_url: endpoint, transport: "chat_completions", default_model: "hermes-fixture", api_key: "no-key-required" } }, + paperclip_auth: { protocol: "chat", style: "none" } }), + }, signal: AbortSignal.timeout(150_000), + }; + const dependencies = { openRuntime: openCodexAcpxRuntime, reportRetainedCleanupFailure: failure => t.diagnostic(String(failure.error)) }; + host = await AcpxRuntimeHost.open(options, dependencies); + const turn = host.startTurn({ text: "Reply with the transport marker.", requestId: "transport-turn" }); + const events = []; + let completed = false; + turn.result.then(() => { completed = true; }); + for await (const event of turn.events) events.push({ event, beforeCompletion: !completed }); + assert.equal((await turn.result).status, "completed"); + const text = events.filter(({ event }) => event.type === "text_delta" && event.stream !== "thought"); + assert.equal(text.map(({ event }) => event.text).join(""), "Hermes transport works."); + assert.ok(text.some(row => row.beforeCompletion)); + assert.ok(requests.some(row => row.body.model === "hermes-fixture")); + assert.ok(requests.every(row => row.authorization === undefined), `No-auth connection sent credentials: ${JSON.stringify(requests.map(row => ({ path: row.path, authenticated: row.authorization !== undefined })))}`); + assert.ok(events.some(({ event }) => event.type === "text_delta" && event.stream === "thought")); + const expectedIdentity = acpxProviderSessionIdentity(host.identity(), host.binding()); + await host.close({ reason: "per-turn process checkpoint" }); + host = await AcpxRuntimeHost.open({ ...options, expectedIdentity }, dependencies); + const imageData = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aA1sAAAAASUVORK5CYII="; + const resumed = host.startTurn({ text: "Continue the previous conversation and inspect this image.", requestId: "resumed-turn", + attachments: [{ schema: "paperclip.user_attachment.v1", kind: "image", name: "pixel.png", mediaType: "image/png", data: imageData }] }); + const resumedEvents = []; + for await (const event of resumed.events) resumedEvents.push(event); + assert.equal(resumedEvents.filter(event => event.type === "text_delta" && event.stream !== "thought").map(event => event.text).join(""), "Hermes transport works.", "Restored history was emitted as new assistant output"); + assert.equal((await resumed.result).status, "completed"); + assert.ok(requests.some(row => row.body.messages?.filter(message => message.role === "user").length >= 2), "Restore lost the previous conversation"); + assert.ok(requests.some(row => row.body.messages?.some(message => Array.isArray(message.content) && message.content.some(part => part.type === "image_url" && part.image_url.url === `data:image/png;base64,${imageData}`))), `The selected model did not receive the authorized image bytes: ${JSON.stringify(requests.filter(row => row.body.stream).map(row => row.body.messages.filter(message => message.role === "user")))}`); + const tools = host.startTurn({ text: "NATIVE_TOOL_FIXTURE: Run the supplied command.", requestId: "tools-turn" }); + const toolEvents = []; + for await (const event of tools.events) toolEvents.push(event); + assert.equal((await tools.result).status, "completed"); + assert.equal(await readFile(join(workspace, "command-proof.txt"), "utf8").catch(() => null), "native-hermes-command\n", + `Native command failed: ${JSON.stringify(requests.flatMap(row => row.body.messages?.filter(message => message.role === "tool") ?? []))}`); + assert.ok(toolEvents.some(event => event.type === "tool_call" || event.type === "tool_call_update"), "Native tool events did not reach ACPX"); + let questions = 0; + const questionTurn = host.startTurn({ text: "NATIVE_QUESTION_FIXTURE: Ask the native question.", requestId: "question-turn", + onExtensionRequest: async (method, params) => { + assert.equal(method, "_hermes/ask_questions"); + assert.equal(params.version, 1); + questions++; + const question = params.input.questions[0]; + assert.ok(question.options[0].label.startsWith("Cobalt")); + return { outcome: "answered", answers: { [question.id]: { selectedOptionIds: ["o0"] } } }; + }, + }); + for await (const _event of questionTurn.events) { /* drain */ } + assert.equal((await questionTurn.result).status, "completed"); + assert.equal(questions, 1, `Native clarify failed: ${JSON.stringify(requests.at(-1)?.body.messages?.filter(message => message.role === "tool"))}`); + assert.ok(requests.some(row => row.body.messages?.some(message => message.role === "tool" && String(message.content).includes("Cobalt"))), "The native callback did not receive the submitted answer"); + for (const kind of ["MCP", "MEMORY"]) { + const turn = host.startTurn({ text: `NATIVE_${kind}_FIXTURE: Use the native tool.`, requestId: `${kind.toLowerCase()}-turn` }); + for await (const _event of turn.events) { /* drain */ } + assert.equal((await turn.result).status, "completed"); + } + assert.deepEqual(bridgeCalls, [{ tool: "fixture_probe", arguments: { marker: "assigned-tool-roundtrip" } }]); + assert.equal(host.steeringCapability()?.steering, true); + assert.equal(host.steeringCapability()?.queuedFollowUp, false); + const steering = host.startTurn({ text: "NATIVE_STEER_FIXTURE: Continue working until redirected.", requestId: "steering-turn" }); + let steered = false; + for await (const event of steering.events) { + if (!steered && event.type === "text_delta") { + await host.steerActiveTurn("NATIVE_STEER_ACCEPTED: Answer with the transport marker instead."); + steered = true; + } + } + assert.equal(steered, true); + assert.equal((await steering.result).status, "completed"); + assert.ok(requests.some(row => JSON.stringify(row.body.messages ?? []).includes("NATIVE_STEER_ACCEPTED")), "Accepted steering did not reach the native turn"); + await assert.rejects(host.steerActiveTurn("stale correction"), /active turn/); + const stop = host.startTurn({ text: "NATIVE_STOP_FIXTURE: Continue working until cancelled.", requestId: "stop-turn" }); + let stopped = false; + for await (const event of stop.events) { + if (!stopped && event.type === "text_delta") { + stopped = true; + await host.interruptActiveTurn("qualification stop"); + } + } + assert.equal(stopped, true); + assert.equal((await stop.result).status, "cancelled"); + const stoppedIdentity = acpxProviderSessionIdentity(host.identity(), host.binding()); + await host.close({ reason: "restore cancelled turn" }); + host = await AcpxRuntimeHost.open({ ...options, expectedIdentity: stoppedIdentity }, dependencies); + const afterStop = host.startTurn({ text: "Reply with a fresh transport marker after cancellation.", requestId: "after-stop-turn" }); + const afterStopEvents = []; + for await (const event of afterStop.events) afterStopEvents.push(event); + const afterStopResult = await afterStop.result; + assert.equal(afterStopResult.status, "completed", JSON.stringify(afterStopResult)); + assert.equal(afterStopEvents.filter(event => event.type === "text_delta" && event.stream !== "thought").map(event => event.text).join(""), "Hermes transport works.", "Cancelled history or old tool results polluted the restored turn"); + assert.ok(!afterStopEvents.some(event => event.type === "tool_call" || event.type === "tool_call_update"), "Old tools replayed as active work"); + assert.ok(requests.at(-1).body.messages.some(message => message.role === "user" && JSON.stringify(message.content).includes("NATIVE_STOP_FIXTURE")), "Cancelled turn history was discarded instead of restored"); + const restoreIdentity = acpxProviderSessionIdentity(host.identity(), host.binding()); + await host.close({ reason: "verify missing history" }); host = undefined; + assert.match(await readFile(join(agentFiles, "hermes/memories/MEMORY.md"), "utf8"), /NATIVE_PERSISTED_MEMORY/); + const nativeDatabases = (await readdir(join(root, "runtime"), { recursive: true })).filter(path => path.endsWith("state.db")); + assert.equal(nativeDatabases.length, 1); + await rm(join(root, "runtime", nativeDatabases[0])); + let refused = false; + try { + host = await AcpxRuntimeHost.open({ ...options, expectedIdentity: restoreIdentity }, dependencies); + const missing = host.startTurn({ text: "Continue the lost conversation.", requestId: "missing-history" }); + for await (const _event of missing.events) { /* drain */ } + refused = (await missing.result).status !== "completed"; + } catch (error) { + assert.match(String(error), /restore|history|session|resume/i); + refused = true; + } + assert.equal(refused, true, "Missing native history was silently replaced with a new conversation"); +}); diff --git a/packages/paperclip-runner/src/drivers/acpx/hermes-state.test.ts b/packages/paperclip-runner/src/drivers/acpx/hermes-state.test.ts index c209e59b050..aab09bfa79d 100644 --- a/packages/paperclip-runner/src/drivers/acpx/hermes-state.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/hermes-state.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, mkdir, readFile, readdir, rm, symlink, link, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, readFile, readdir, rename, rm, symlink, link, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -46,6 +46,22 @@ describe("Hermes managed learned state", () => { const f = await fixture(); await expect(stageHermesAgentState(f.home, null)).rejects.toThrow(/managed agent-file storage/); }); + it.each(["symlink", "replacement"])("rejects a %s state parent without changing the redirected files", async kind => { + const f = await fixture(); + const state = await stageHermesAgentState(f.home, f.context); + const outside = join(f.root, "outside"); + await mkdir(outside); await mkdir(join(outside, "memories")); await mkdir(join(outside, "skills")); + await writeFile(join(outside, "memories/MEMORY.md"), "Outside memory"); + await writeFile(join(outside, "memories/keep.md"), "Must not be deleted"); + await writeFile(join(f.home, "memories/MEMORY.md"), "Native change"); + await rename(join(f.agent, "hermes"), join(f.agent, "original-hermes")); + if (kind === "symlink") await symlink(outside, join(f.agent, "hermes")); + else await rename(outside, join(f.agent, "hermes")); + await expect(state.collect()).rejects.toThrow(/unsafe directory|identity changed/); + const redirected = kind === "symlink" ? outside : join(f.agent, "hermes"); + expect(await readFile(join(redirected, "memories/MEMORY.md"), "utf8")).toBe("Outside memory"); + expect(await readFile(join(redirected, "memories/keep.md"), "utf8")).toBe("Must not be deleted"); + }); it("discards unfinished transfers during staging and collection without learning their contents", async () => { const f = await fixture(); const state = await stageHermesAgentState(f.home, f.context); diff --git a/packages/paperclip-runner/src/drivers/acpx/hermes-state.ts b/packages/paperclip-runner/src/drivers/acpx/hermes-state.ts index 0c212f6f13f..7d2d64fd42b 100644 --- a/packages/paperclip-runner/src/drivers/acpx/hermes-state.ts +++ b/packages/paperclip-runner/src/drivers/acpx/hermes-state.ts @@ -18,13 +18,19 @@ export async function stageHermesAgentState(home: string, context: NativeRuntime await assertDirectory(agentRoot); const stateRoot = join(agentRoot, "hermes"); await ensureDirectory(stateRoot); - for (const name of DIRECTORIES) await mirrorDirectory(join(stateRoot, name), join(home, name)); + const agentIdentity = await directoryIdentity(agentRoot); + const stateIdentity = await directoryIdentity(stateRoot); + const assertRoots = async () => { + await assertIdentity(agentRoot, agentIdentity); + await assertIdentity(stateRoot, stateIdentity); + }; + for (const name of DIRECTORIES) await mirrorDirectory(join(stateRoot, name), join(home, name), assertRoots); return { async collect() { // Called after verified process exit and before the controller collects // the existing working copy. Its usual authorization and save receipt apply. - await assertDirectory(agentRoot); - for (const name of DIRECTORIES) await mirrorDirectory(join(home, name), join(stateRoot, name)); + await assertRoots(); + for (const name of DIRECTORIES) await mirrorDirectory(join(home, name), join(stateRoot, name), assertRoots); }, }; } @@ -33,6 +39,15 @@ async function assertDirectory(path: string) { const stat = await lstat(path); if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Hermes state contains an unsafe directory"); } +async function directoryIdentity(path: string) { + await assertDirectory(path); + const stat = await lstat(path, { bigint: true }); + return { dev: stat.dev, ino: stat.ino }; +} +async function assertIdentity(path: string, expected: { dev: bigint; ino: bigint }) { + const actual = await directoryIdentity(path); + if (actual.dev !== expected.dev || actual.ino !== expected.ino) throw new Error("Hermes state directory identity changed"); +} async function ensureDirectory(path: string) { await mkdir(path, { mode: 0o700 }).catch(error => { if (error.code !== "EEXIST") throw error; }); await assertDirectory(path); @@ -64,21 +79,41 @@ async function inventory(root: string): Promise> { await walk(root); return files; } -async function mirrorDirectory(source: string, destination: string) { +async function mirrorDirectory(source: string, destination: string, assertRoots: () => Promise) { + await assertRoots(); await ensureDirectory(source); await ensureDirectory(destination); const files = await inventory(source); const previous = await inventory(destination); + const sourceIdentity = await directoryIdentity(source); + const destinationIdentity = await directoryIdentity(destination); + const assertBoundRoots = async () => { + await assertRoots(); + await assertIdentity(source, sourceIdentity); + await assertIdentity(destination, destinationIdentity); + }; for (const [name, size] of files) { + await assertBoundRoots(); const target = join(destination, name); const parts = relative(destination, dirname(target)).split(/[\\/]/).filter(Boolean); let directory = destination; - for (const part of parts) { directory = join(directory, part); await ensureDirectory(directory); } + const parents: { path: string; identity: { dev: bigint; ino: bigint } }[] = []; + for (const part of parts) { + await assertBoundRoots(); + for (const parent of parents) await assertIdentity(parent.path, parent.identity); + directory = join(directory, part); await ensureDirectory(directory); + parents.push({ path: directory, identity: await directoryIdentity(directory) }); + } + const assertDestination = async () => { + await assertBoundRoots(); + for (const parent of parents) await assertIdentity(parent.path, parent.identity); + }; const input = await open(join(source, name), constants.O_RDONLY | constants.O_NOFOLLOW); const temporary = join(dirname(target), `.paperclip-hermes-transfer-${randomUUID()}.tmp`); try { const before = await input.stat({ bigint: true }); if (!before.isFile() || before.nlink !== 1n || before.size !== BigInt(size)) throw new Error("Hermes state changed during collection"); + await assertDestination(); const output = await open(temporary, "wx", 0o600); try { const buffer = Buffer.alloc(Math.min(size + 1, 64 * 1024)); @@ -93,8 +128,20 @@ async function mirrorDirectory(source: string, destination: string) { } finally { await output.close(); } const after = await input.stat({ bigint: true }); if (before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs || before.size !== after.size) throw new Error("Hermes state changed during collection"); + await assertDestination(); await rename(temporary, target); - } finally { await input.close(); await rm(temporary, { force: true }); } + } finally { + await input.close(); + await assertDestination(); + await rm(temporary, { force: true }); + } + } + for (const name of previous.keys()) if (!files.has(name)) { + await assertBoundRoots(); + let directory = destination; + for (const part of relative(destination, dirname(join(destination, name))).split(/[\\/]/).filter(Boolean)) { + directory = join(directory, part); await assertDirectory(directory); + } + await rm(join(destination, name)); } - for (const name of previous.keys()) if (!files.has(name)) await rm(join(destination, name)); } diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 72de187800e..78c4cfad339 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1461,17 +1461,14 @@ immutable Daytona image; no private hooks or fixture database writes are used. ## Extended ACP harnesses (explicit only) -`--suite extended-harnesses` declares 40 Product E2E cells: Cursor, Copilot, -Pi and Hermes on local and Daytona, each exercising authenticated completion, +`--suite extended-harnesses` declares 30 Product E2E cells: Cursor, Copilot, +and Pi on local and Daytona, each exercising authenticated completion, question/answer continuation, revision-bound semantic plan approval, restart with pending input, and file edit plus independent byte validation. The file case uses a public project workspace so Daytona copy-back is graded too. These are candidate definitions, not a claim of provider qualification. Native provider-specific questions, plan decisions, restrictive permissions and steering need their separate conformance/qualification evidence. -Hermes uses an existing managed OpenRouter connection through the ordinary -connection fixture. It remains pending qualification; the native transport -fixture does not substitute for these browser and remote workflows. ```sh pnpm test:e2e:runner -- --list --suite extended-harnesses diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 5ac24945c5a..6b386d6e182 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -146,10 +146,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 63, 12, 6, 8, 2, 2, 2, 40, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, + 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, ]); - expect(validateRunnerCatalog()).toHaveLength(708); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(708); + expect(validateRunnerCatalog()).toHaveLength(698); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(698); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 4358aab7d46..d9ddf5300e2 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -174,7 +174,7 @@ function nativeProfile(input: { provider: "codex" | "opencode" | "acpx"; model: string; credential: RunnerProfileFixture["credential"]; - acpxAgent?: "claude" | "codex" | "grok" | "cursor" | "copilot" | "pi" | "hermes"; + acpxAgent?: "claude" | "codex" | "grok" | "cursor" | "copilot" | "pi"; qualificationCandidate?: RunnerProfileFixture["qualificationCandidate"]; supportedEnvironments?: readonly (typeof ENVIRONMENT_IDS)[number][]; modelQualification?: RunnerProfileFixture["modelQualification"]; @@ -338,11 +338,6 @@ export const runnerProfiles: readonly RunnerProfileFixture[] = [ // Explicit qualification choices from authenticated model discovery, not shipped // defaults. Cursor has local/Daytona proof; Copilot and Pi remain pending. export const extendedHarnessProfiles: readonly RunnerProfileFixture[] = [ - nativeProfile({ - id: "runner-acpx-hermes", label: "Runner Hermes (candidate)", provider: "acpx", acpxAgent: "hermes", - qualificationCandidate: "hermes", credential: "OPENROUTER_API_KEY", model: "deepseek/deepseek-v4-flash-0731", - modelQualification: { source: "candidate_runner_profile", qualificationId: "hermes:v2026.9.24:openrouter:pending" }, - }), nativeProfile({ id: "runner-acpx-cursor", label: "Runner Cursor", provider: "acpx", acpxAgent: "cursor", qualificationCandidate: "cursor", credential: "CURSOR_AUTH_TOKEN", @@ -1185,7 +1180,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Explicit candidate qualification through real Paperclip tools, browser interactions, file edits and restart recovery.", groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments, tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask], - expectedMatrixSize: 40, + expectedMatrixSize: 30, definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28" }, }, { diff --git a/tests/runner-e2e/extended-harnesses.test.ts b/tests/runner-e2e/extended-harnesses.test.ts index be0d05d85c6..73abc204be3 100644 --- a/tests/runner-e2e/extended-harnesses.test.ts +++ b/tests/runner-e2e/extended-harnesses.test.ts @@ -7,7 +7,7 @@ import { findSecretLeak, redactText } from "./redaction.js"; const selected = runnerMatrix.filter(cell => cell.suite.id === "extended-harnesses"); describe("extended ACP harness qualification", () => { it("has exactly five real product journeys per provider in both environments, excluded from all", () => { - expect(selected).toHaveLength(40); + expect(selected).toHaveLength(30); expect(new Set(selected.map(cell => cell.environment.id))).toEqual(new Set(["local", "daytona"])); expect(new Set(selected.map(cell => cell.task.id))).toEqual(new Set([ "hello-complete", "question-resume-complete", "plan-approve-complete", "structured-question-restart-resume", "file-edit-validate", @@ -18,7 +18,6 @@ describe("extended ACP harness qualification", () => { }); it("uses exact discovered models, encrypted credential references and current qualification metadata", () => { expect(extendedHarnessProfiles.map(profile => profile.model)).toEqual([ - "deepseek/deepseek-v4-flash-0731", "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]", "gpt-5.6-luna", "openrouter/deepseek/deepseek-v4-flash-0731", ]); for (const profile of extendedHarnessProfiles) { @@ -29,7 +28,7 @@ describe("extended ACP harness qualification", () => { } }); it("replaces ambient admission with the selected exact pairs and strips raw provider credentials from server", () => { - const cell = selected.find(cell => cell.profile.qualificationCandidate === "cursor")!; + const cell = selected[0]!; const ambient = { PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient", CURSOR_AUTH_TOKEN: "cursor", CURSOR_API_KEY: "cursor-key", COPILOT_GITHUB_TOKEN: "copilot", GH_TOKEN: "github", GITHUB_TOKEN: "github" }; const env = buildRunnerE2EProcessEnvironment(ambient, [cell]); expect(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index 2bce9d3e013..d3ac65d6d79 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -92,7 +92,6 @@ describe("live runner fixtures", () => { }); it.each([ - ["runner-acpx-hermes", "extended-harnesses", "hello-complete"], ["runner-codex", "hiring-templates", "hire-coder-template-reuse"], ["runner-acpx-claude", "hiring-templates", "hire-coder-template-reuse"], ["runner-codex", "everyday-workflows", "hire-reuse"], @@ -111,7 +110,7 @@ describe("live runner fixtures", () => { e.environment.id === "local", )!; const provider = - profile === "runner-acpx-claude" ? "anthropic" : ["runner-opencode", "runner-acpx-hermes"].includes(profile) ? "openrouter" : "openai"; + profile === "runner-acpx-claude" ? "anthropic" : profile === "runner-opencode" ? "openrouter" : "openai"; let connected = false; let agentBody: any; const api = { diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index ad22b4165e1..26512d6079a 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -241,8 +241,7 @@ export async function setupLiveFixtures(input: { }, }); - const managedHiring = isManagedHiringCase(execution.suite.id, execution.task.id) - || execution.profile.qualificationCandidate === "hermes"; + const managedHiring = isManagedHiringCase(execution.suite.id, execution.task.id); if (managedHiring) { registry.register({ id: "ai-connection", diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 0f0ad2328f9..4a88ef5ca2f 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -78,7 +78,7 @@ export interface RunnerProfileFixture { | "openrouter_rankings_snapshot"; qualificationId: string; }; - qualificationCandidate?: "cursor" | "copilot" | "pi" | "hermes"; + qualificationCandidate?: "cursor" | "copilot" | "pi"; ranking?: { rank: number; canonicalModelId: string;