mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 16:27:18 +02:00
fix(hermes): keep native candidate independently restorable
Reject replaced learned-state parents, include authenticated run attachment and the advertised native transport fixtures in this candidate. Keep paid browser campaign definitions in the qualification PR. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
5ca8dc587c
commit
772db06812
13 files changed
+423
-38
No files matched your search
@@ -3266,9 +3266,15 @@ mod tests {
|
||||
authenticated_run_grant_attachment("claude");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn hermes_attachment_rotates_only_authenticated_run_grants() {
|
||||
authenticated_run_grant_attachment("hermes");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn authenticated_run_grant_attachment(agent: &str) {
|
||||
let directory = temporary_directory("cursor-cross-run-attach");
|
||||
let directory = temporary_directory(&format!("{agent}-cross-run-attach"));
|
||||
let runtime = directory.join("runtime");
|
||||
let workspace = directory.join("workspace");
|
||||
fs::create_dir_all(&runtime).unwrap();
|
||||
|
||||
+8
-6
@@ -11,7 +11,7 @@ pub(crate) enum RunAttachmentPolicy {
|
||||
|
||||
pub(crate) fn run_attachment_policy(agent: &str) -> RunAttachmentPolicy {
|
||||
match agent {
|
||||
"cursor" => RunAttachmentPolicy::AuthenticatedRunGrants,
|
||||
"cursor" | "hermes" => RunAttachmentPolicy::AuthenticatedRunGrants,
|
||||
"claude" => RunAttachmentPolicy::AuthenticatedAssetPaths,
|
||||
_ => RunAttachmentPolicy::ImmutableInstructions,
|
||||
}
|
||||
@@ -22,11 +22,13 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn only_qualified_providers_opt_into_run_grant_rotation() {
|
||||
assert_eq!(
|
||||
run_attachment_policy("cursor"),
|
||||
RunAttachmentPolicy::AuthenticatedRunGrants
|
||||
);
|
||||
fn only_explicit_profiles_opt_into_run_grant_rotation() {
|
||||
for agent in ["cursor", "hermes"] {
|
||||
assert_eq!(
|
||||
run_attachment_policy(agent),
|
||||
RunAttachmentPolicy::AuthenticatedRunGrants
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
run_attachment_policy("claude"),
|
||||
RunAttachmentPolicy::AuthenticatedAssetPaths
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
// Production TS -> Rust PRP -> sidecar -> ACPX -> pinned Hermes -> deterministic
|
||||
// HTTP model fixture. This is not live provider or browser qualification.
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { once } from 'node:events';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { createServer } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { createRunnerdNativeSessionBackend } from '../dist/backends/codex-native-backend.js';
|
||||
import { createRunnerdCodexTransport } from '../dist/live/runnerd-codex-transport.js';
|
||||
import { resolveQualifiedAcpxProfile } from '../dist/drivers/acpx/qualified-profiles.js';
|
||||
import { parseNativeExecutionInput } from '../dist/contracts/native-execution.js';
|
||||
import { PAPERCLIP_EXECUTION_PROMPT, PAPERCLIP_EXECUTION_PROMPT_REVISION, nativeRuntimePromptDigest, canonicalNativeRuntimeContextDigest } from '../dist/contracts/runtime-context.js';
|
||||
|
||||
const sha256 = text => createHash('sha256').update(text).digest('hex');
|
||||
const imageData = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aA1sAAAAASUVORK5CYII=';
|
||||
const completion = {
|
||||
reportedWorkDisposition: 'done', summary: 'Native Rust path completed.',
|
||||
completionClaim: { contractRevision: '1', objectiveSatisfied: true,
|
||||
criteria: [{ criterionId: 'objective', status: 'satisfied', evidenceRefs: [] }], remainingWork: [] },
|
||||
evidence: [], verification: [{ commandOrCheck: 'native transport fixture', status: 'passed' }], attentionRequests: [], artifacts: [],
|
||||
};
|
||||
|
||||
test('pinned Hermes streams images and semantic completion through Rust PRP and the production sidecar', {
|
||||
skip: process.env.PAPERCLIP_HERMES_QUALIFY !== '1', timeout: 180_000,
|
||||
}, async t => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'paperclip-hermes-runnerd-'));
|
||||
const workspace = join(root, 'workspace');
|
||||
const instructions = join(root, 'instructions');
|
||||
const agentFiles = join(root, 'agent-files');
|
||||
for (const directory of [workspace, instructions, agentFiles, join(root, 'runtime')]) await mkdir(directory, { mode: 0o700 });
|
||||
const entry = 'You are a deterministic native transport test agent.';
|
||||
await writeFile(join(instructions, 'AGENTS.md'), entry);
|
||||
const requests = [];
|
||||
const server = createServer(async (req, res) => {
|
||||
const chunks = []; for await (const chunk of req) chunks.push(chunk);
|
||||
const body = JSON.parse(Buffer.concat(chunks).toString() || '{}');
|
||||
requests.push(body);
|
||||
assert.equal(req.headers.authorization, undefined);
|
||||
if (req.url !== '/v1/chat/completions') { res.writeHead(404).end(); return; }
|
||||
if (!body.stream) {
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({ id: 'title', object: 'chat.completion', model: 'hermes-fixture', choices: [{ index: 0, message: { role: 'assistant', content: 'Fixture' }, finish_reason: 'stop' }] }));
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { 'content-type': 'text/event-stream' });
|
||||
const send = (delta, finish_reason = null) => res.write(`data: ${JSON.stringify({ id: 'native-prp', object: 'chat.completion.chunk', choices: [{ index: 0, delta, finish_reason }] })}\n\n`);
|
||||
if (!body.messages.some(message => message.role === 'tool')) {
|
||||
send({ role: 'assistant', reasoning_content: 'Checking the native runner path.' });
|
||||
await new Promise(resolve => setTimeout(resolve, 80));
|
||||
const finish = body.tools.find(tool => tool.function.name.endsWith('paperclip_finish'));
|
||||
assert.ok(finish, 'Paperclip completion authority was not exposed to the native model');
|
||||
send({ tool_calls: [{ index: 0, id: 'native-prp-finish', type: 'function', function: { name: finish.function.name, arguments: JSON.stringify(completion) } }] });
|
||||
send({}, 'tool_calls');
|
||||
} else {
|
||||
send({ role: 'assistant', content: 'Native Rust ' });
|
||||
await new Promise(resolve => setTimeout(resolve, 80));
|
||||
send({ content: 'path completed.' });
|
||||
send({}, 'stop');
|
||||
}
|
||||
res.end('data: {"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}\n\ndata: [DONE]\n\n');
|
||||
});
|
||||
let session, bundle;
|
||||
t.after(async () => {
|
||||
await session?.close({ reason: 'fixture cleanup' });
|
||||
await bundle?.transport.close();
|
||||
server.closeAllConnections(); await new Promise(resolve => server.close(resolve));
|
||||
if (process.env.PAPERCLIP_KEEP_HERMES_FIXTURE === '1') t.diagnostic(`Fixture evidence: ${root}`);
|
||||
else await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
server.listen(0, '127.0.0.1'); await once(server, 'listening');
|
||||
const context = {
|
||||
prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() },
|
||||
instructions: { entryPath: 'AGENTS.md', bundle: { schema: 'paperclip.runtime-asset.v1', digest: sha256(entry), manifestDigest: sha256(entry), rootPath: instructions, fileCount: 1, totalBytes: Buffer.byteLength(entry) },
|
||||
workingCopy: { kind: 'agent_files', rootPath: agentFiles, entryPath: 'AGENTS.md' } },
|
||||
skills: [], mcp: { assignmentSetId: 'none', digest: sha256('none'), bindingId: null },
|
||||
};
|
||||
const runtimeContext = { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) };
|
||||
const identity = { runId: 'hermes-prp-run', sessionId: 'hermes-prp-session', companyId: 'hermes-prp-company', issueId: 'hermes-prp-issue', agentId: 'hermes-prp-agent' };
|
||||
const resolved = resolveQualifiedAcpxProfile('hermes', 'hermes-fixture');
|
||||
const profile = Object.fromEntries(['driverKind', 'protocolVersion', 'acpxVersion', 'agent', 'agentProfileVersion', 'agentServerPackage', 'agentServerVersion', 'agentRuntimePackage', 'agentRuntimeVersion', 'commandDigest'].map(key => [key, resolved[key]]));
|
||||
const input = parseNativeExecutionInput({
|
||||
schema: 'paperclip.native-execution-input.v6', executionMode: 'default', planningContext: null,
|
||||
binding: { runId: identity.runId, companyId: identity.companyId, issueId: identity.issueId, agentId: identity.agentId, executionWorkspaceId: 'hermes-prp-workspace' },
|
||||
session: { normalizedSessionId: identity.sessionId, driverKind: 'acpx_runtime', protocolVersion: 1 },
|
||||
task: { identifier: 'HERMES-PRP', title: 'Native transport fixture', description: null, prompt: 'Complete the native transport fixture.', workMode: 'standard' },
|
||||
workspace: { cwd: workspace, repoUrl: null, repoRef: null, branchName: null },
|
||||
provider: { kind: 'acpx', agent: 'hermes', model: 'hermes-fixture', permissionMode: 'approve-all', profile, connectionFingerprint: '1'.repeat(64) },
|
||||
completionContract: { id: 'hermes-prp-contract', sha256: sha256('fixture'), schemaVersion: 'paperclip.completion-contract.v1', contract: { revision: '1', objective: 'Native transport fixture', criteria: [{ id: 'objective', requirement: 'Complete the native transport fixture.' }] } },
|
||||
runtimeContext, interactionResponses: [], credentialBindings: [],
|
||||
attachments: [{ schema: 'paperclip.user_attachment.v1', kind: 'image', name: 'pixel.png', mediaType: 'image/png', data: imageData }],
|
||||
});
|
||||
bundle = createRunnerdCodexTransport({
|
||||
provider: 'acpx', acpxAgent: 'hermes', acpxCandidateProfile: 'hermes', acpxPermissionMode: 'approve-all',
|
||||
stateDirectory: join(root, 'runner'), acpxRuntimeDirectory: join(root, 'runtime'), runtimeContext,
|
||||
lifecyclePolicy: { mode: 'per_turn', idleTimeoutMs: null }, turnStartTimeoutMs: 90_000,
|
||||
onDiagnostic: message => t.diagnostic(message),
|
||||
environment: { PATH: process.env.PATH, PAPERCLIP_PROVIDER_TRACE_PATH: join(root, 'provider-trace.jsonl'), PAPERCLIP_HERMES_CONNECTION_FINGERPRINT: input.provider.connectionFingerprint,
|
||||
PAPERCLIP_HERMES_CONFIG_JSON: JSON.stringify({ model: { provider: 'custom:paperclip', default: 'hermes-fixture' },
|
||||
providers: { paperclip: { base_url: `http://127.0.0.1:${server.address().port}/v1`, transport: 'chat_completions', default_model: 'hermes-fixture', api_key: 'no-key-required' } },
|
||||
paperclip_auth: { protocol: 'chat', style: 'none' } }),
|
||||
},
|
||||
prpIdentity: { runnerInstanceId: 'hermes-prp-runner', environmentLeaseId: 'hermes-prp-lease', runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: 'hermes-prp-turn', itemId: 'hermes-prp-item' },
|
||||
});
|
||||
const backend = createRunnerdNativeSessionBackend(input, { runnerInstanceId: 'hermes-prp-runner', transportFactory: () => bundle.transport });
|
||||
session = await backend.openSession({ identity, workingDirectory: workspace });
|
||||
await session.startTurn({ message: { role: 'user', text: input.task.prompt, attachments: input.attachments } });
|
||||
const events = [];
|
||||
for await (const event of session.events()) {
|
||||
events.push(event);
|
||||
if (['run.terminal', 'turn.completed', 'turn.failed', 'turn.cancelled', 'turn.interrupted'].includes(event.eventType)) break;
|
||||
}
|
||||
assert.ok(events.some(event => event.eventType === 'turn.completed'), JSON.stringify(events));
|
||||
assert.ok(events.some(event => JSON.stringify(event.payload).includes('Checking the native runner path.')), 'Reasoning did not cross PRP');
|
||||
assert.ok(events.some(event => JSON.stringify(event.payload).includes('Native Rust')), 'Text did not cross PRP');
|
||||
assert.ok(requests.some(body => body.messages?.some(message => Array.isArray(message.content) && message.content.some(part => part.type === 'image_url' && part.image_url.url === `data:image/png;base64,${imageData}`))), 'Images did not cross TS/Rust/sidecar');
|
||||
const snapshot = await session.snapshot();
|
||||
assert.equal(snapshot.semanticResult?.reportedWorkDisposition, 'done');
|
||||
assert.equal(snapshot.terminal?.runTerminalState, 'succeeded');
|
||||
assert.equal(bundle.evidence().acpxAgent, 'hermes');
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
// Pinned native Hermes + production ACPX host, deterministic model fixture.
|
||||
// This is transport proof, not live model or product qualification.
|
||||
// PAPERCLIP_HERMES_QUALIFY=1 node --test scripts/qualify-hermes-runtime.test.mjs
|
||||
import assert from "node:assert/strict";
|
||||
import { once } from "node:events";
|
||||
import { mkdir, mkdtemp, rm, readFile, readdir } from "node:fs/promises";
|
||||
import { createServer } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { AcpxRuntimeHost } from "../dist/drivers/acpx/runtime-host.js";
|
||||
import { openCodexAcpxRuntime } from "../dist/drivers/acpx/codex-runtime-adapter.js";
|
||||
import { acpxProfileClientCapabilities } from "../dist/drivers/acpx/profile-extensions.js";
|
||||
import { acpxProviderSessionIdentity } from "../dist/drivers/acpx/recovery-identity.js";
|
||||
|
||||
test("pinned Hermes streams through the production ACPX host using a no-auth local connection", {
|
||||
skip: process.env.PAPERCLIP_HERMES_QUALIFY !== "1", timeout: 180_000,
|
||||
}, async (t) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-hermes-transport-"));
|
||||
const workspace = join(root, "workspace");
|
||||
const agentFiles = join(root, "agent-files");
|
||||
await mkdir(workspace); await mkdir(agentFiles); await mkdir(join(root, "runtime"));
|
||||
let host;
|
||||
const requests = [];
|
||||
const bridgeCalls = [];
|
||||
const server = createServer(async (req, res) => {
|
||||
const chunks = [];
|
||||
for await (const chunk of req) chunks.push(chunk);
|
||||
const body = JSON.parse(Buffer.concat(chunks).toString() || "{}");
|
||||
requests.push({ path: req.url, authorization: req.headers.authorization, body });
|
||||
if (req.url !== "/v1/chat/completions") { res.writeHead(404).end(); return; }
|
||||
if (!body.stream) {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ id: "fixture", object: "chat.completion", model: "hermes-fixture", choices: [{ index: 0, message: { role: "assistant", content: "Fixture title" }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 } }));
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { "content-type": "text/event-stream" });
|
||||
const lastUser = body.messages.findLastIndex(message => message.role === "user");
|
||||
const userText = JSON.stringify(body.messages[lastUser]?.content);
|
||||
if ((userText.includes("NATIVE_STEER_FIXTURE") && !JSON.stringify(body.messages).includes("NATIVE_STEER_ACCEPTED")) || userText.includes("NATIVE_STOP_FIXTURE")) {
|
||||
for (let index = 0; index < 150 && !res.destroyed; index++) {
|
||||
res.write(`data: ${JSON.stringify({ id: "control-fixture", choices: [{ index: 0, delta: { reasoning_content: "Working. " }, finish_reason: null }] })}\n\n`);
|
||||
await new Promise(resolve => setTimeout(resolve, 30));
|
||||
}
|
||||
if (!res.destroyed) res.end('data: {"id":"control-fixture","choices":[{"index":0,"delta":{"content":"Control fixture complete."},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n');
|
||||
return;
|
||||
}
|
||||
const toolDone = body.messages.slice(lastUser + 1).some(message => message.role === "tool");
|
||||
if (!toolDone && (userText.includes("NATIVE_TOOL_FIXTURE") || userText.includes("NATIVE_QUESTION_FIXTURE") || userText.includes("NATIVE_MCP_FIXTURE") || userText.includes("NATIVE_MEMORY_FIXTURE"))) {
|
||||
const question = userText.includes("NATIVE_QUESTION_FIXTURE");
|
||||
const bridge = userText.includes("NATIVE_MCP_FIXTURE");
|
||||
const memory = userText.includes("NATIVE_MEMORY_FIXTURE");
|
||||
const name = question ? "clarify" : memory ? "memory" : bridge ? body.tools.find(tool => tool.function?.name.endsWith("fixture_probe"))?.function.name ?? "missing_assigned_tool" : "terminal";
|
||||
const args = question ? { question: "Choose the fixture color", choices: ["Cobalt", "Amber"] }
|
||||
: memory ? { action: "add", target: "memory", content: "NATIVE_PERSISTED_MEMORY: Project uses Cobalt." }
|
||||
: bridge ? { marker: "assigned-tool-roundtrip" }
|
||||
: { command: "printf 'native-hermes-command\\n' > command-proof.txt", timeout: 10 };
|
||||
res.write(`data: ${JSON.stringify({ id: "tool-fixture", object: "chat.completion.chunk", model: "hermes-fixture", created: 1, choices: [{ index: 0, delta: { role: "assistant", tool_calls: [{ index: 0, id: `native-${name}`, type: "function", function: { name, arguments: JSON.stringify(args) } }] }, finish_reason: null }] })}\n\n`);
|
||||
res.end(`data: ${JSON.stringify({ id: "tool-fixture", choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } })}\n\ndata: [DONE]\n\n`);
|
||||
return;
|
||||
}
|
||||
for (const [delta, finish_reason] of [
|
||||
[{ role: "assistant", reasoning_content: "Fixture reasoning." }, null],
|
||||
[{ content: "Hermes " }, null], [{ content: "transport works." }, null], [{}, "stop"],
|
||||
]) {
|
||||
res.write(`data: ${JSON.stringify({ id: "fixture", object: "chat.completion.chunk", model: "hermes-fixture", created: 1, choices: [{ index: 0, delta, finish_reason }] })}\n\n`);
|
||||
await new Promise(resolve => setTimeout(resolve, 40));
|
||||
}
|
||||
res.end('data: {"id":"fixture","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}\n\ndata: [DONE]\n\n');
|
||||
});
|
||||
t.after(async () => {
|
||||
if (host) await host.close({ reason: "qualification cleanup" });
|
||||
server.closeAllConnections(); await new Promise(resolve => server.close(resolve));
|
||||
if (process.env.PAPERCLIP_HERMES_QUALIFY_KEEP === "1") t.diagnostic(`Private fixture evidence: ${root}`);
|
||||
else await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
server.listen(0, "127.0.0.1"); await once(server, "listening");
|
||||
const endpoint = `http://127.0.0.1:${server.address().port}/v1`;
|
||||
const options = {
|
||||
runtimeDirectory: join(root, "runtime"), normalizedSessionId: "hermes-transport-test", workingDirectory: workspace,
|
||||
agent: "hermes", model: "hermes-fixture", permissionMode: "approve-all", providerPolicy: { readOnly: false },
|
||||
clientCapabilities: acpxProfileClientCapabilities("hermes"),
|
||||
systemInstructions: "This is a transport fixture. Reply briefly.",
|
||||
semanticTools: {
|
||||
tools: [{ name: "fixture_probe", description: "Return the supplied marker.", inputSchema: { type: "object", properties: { marker: { type: "string" } }, required: ["marker"], additionalProperties: false } }],
|
||||
handler: async call => { bridgeCalls.push({ tool: call.tool, arguments: call.arguments }); return { marker: "assigned-tool-returned" }; },
|
||||
},
|
||||
runtimeContext: { instructions: { workingCopy: { kind: "agent_files", rootPath: agentFiles, entryPath: "AGENTS.md" } }, skills: [], mcp: {} },
|
||||
environment: { PATH: process.env.PATH,
|
||||
PAPERCLIP_HERMES_CONNECTION_FINGERPRINT: "1".repeat(64),
|
||||
PAPERCLIP_HERMES_CONFIG_JSON: JSON.stringify({ model: { provider: "custom:paperclip", default: "hermes-fixture" },
|
||||
providers: { paperclip: { base_url: endpoint, transport: "chat_completions", default_model: "hermes-fixture", api_key: "no-key-required" } },
|
||||
paperclip_auth: { protocol: "chat", style: "none" } }),
|
||||
}, signal: AbortSignal.timeout(150_000),
|
||||
};
|
||||
const dependencies = { openRuntime: openCodexAcpxRuntime, reportRetainedCleanupFailure: failure => t.diagnostic(String(failure.error)) };
|
||||
host = await AcpxRuntimeHost.open(options, dependencies);
|
||||
const turn = host.startTurn({ text: "Reply with the transport marker.", requestId: "transport-turn" });
|
||||
const events = [];
|
||||
let completed = false;
|
||||
turn.result.then(() => { completed = true; });
|
||||
for await (const event of turn.events) events.push({ event, beforeCompletion: !completed });
|
||||
assert.equal((await turn.result).status, "completed");
|
||||
const text = events.filter(({ event }) => event.type === "text_delta" && event.stream !== "thought");
|
||||
assert.equal(text.map(({ event }) => event.text).join(""), "Hermes transport works.");
|
||||
assert.ok(text.some(row => row.beforeCompletion));
|
||||
assert.ok(requests.some(row => row.body.model === "hermes-fixture"));
|
||||
assert.ok(requests.every(row => row.authorization === undefined), `No-auth connection sent credentials: ${JSON.stringify(requests.map(row => ({ path: row.path, authenticated: row.authorization !== undefined })))}`);
|
||||
assert.ok(events.some(({ event }) => event.type === "text_delta" && event.stream === "thought"));
|
||||
const expectedIdentity = acpxProviderSessionIdentity(host.identity(), host.binding());
|
||||
await host.close({ reason: "per-turn process checkpoint" });
|
||||
host = await AcpxRuntimeHost.open({ ...options, expectedIdentity }, dependencies);
|
||||
const imageData = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aA1sAAAAASUVORK5CYII=";
|
||||
const resumed = host.startTurn({ text: "Continue the previous conversation and inspect this image.", requestId: "resumed-turn",
|
||||
attachments: [{ schema: "paperclip.user_attachment.v1", kind: "image", name: "pixel.png", mediaType: "image/png", data: imageData }] });
|
||||
const resumedEvents = [];
|
||||
for await (const event of resumed.events) resumedEvents.push(event);
|
||||
assert.equal(resumedEvents.filter(event => event.type === "text_delta" && event.stream !== "thought").map(event => event.text).join(""), "Hermes transport works.", "Restored history was emitted as new assistant output");
|
||||
assert.equal((await resumed.result).status, "completed");
|
||||
assert.ok(requests.some(row => row.body.messages?.filter(message => message.role === "user").length >= 2), "Restore lost the previous conversation");
|
||||
assert.ok(requests.some(row => row.body.messages?.some(message => Array.isArray(message.content) && message.content.some(part => part.type === "image_url" && part.image_url.url === `data:image/png;base64,${imageData}`))), `The selected model did not receive the authorized image bytes: ${JSON.stringify(requests.filter(row => row.body.stream).map(row => row.body.messages.filter(message => message.role === "user")))}`);
|
||||
const tools = host.startTurn({ text: "NATIVE_TOOL_FIXTURE: Run the supplied command.", requestId: "tools-turn" });
|
||||
const toolEvents = [];
|
||||
for await (const event of tools.events) toolEvents.push(event);
|
||||
assert.equal((await tools.result).status, "completed");
|
||||
assert.equal(await readFile(join(workspace, "command-proof.txt"), "utf8").catch(() => null), "native-hermes-command\n",
|
||||
`Native command failed: ${JSON.stringify(requests.flatMap(row => row.body.messages?.filter(message => message.role === "tool") ?? []))}`);
|
||||
assert.ok(toolEvents.some(event => event.type === "tool_call" || event.type === "tool_call_update"), "Native tool events did not reach ACPX");
|
||||
let questions = 0;
|
||||
const questionTurn = host.startTurn({ text: "NATIVE_QUESTION_FIXTURE: Ask the native question.", requestId: "question-turn",
|
||||
onExtensionRequest: async (method, params) => {
|
||||
assert.equal(method, "_hermes/ask_questions");
|
||||
assert.equal(params.version, 1);
|
||||
questions++;
|
||||
const question = params.input.questions[0];
|
||||
assert.ok(question.options[0].label.startsWith("Cobalt"));
|
||||
return { outcome: "answered", answers: { [question.id]: { selectedOptionIds: ["o0"] } } };
|
||||
},
|
||||
});
|
||||
for await (const _event of questionTurn.events) { /* drain */ }
|
||||
assert.equal((await questionTurn.result).status, "completed");
|
||||
assert.equal(questions, 1, `Native clarify failed: ${JSON.stringify(requests.at(-1)?.body.messages?.filter(message => message.role === "tool"))}`);
|
||||
assert.ok(requests.some(row => row.body.messages?.some(message => message.role === "tool" && String(message.content).includes("Cobalt"))), "The native callback did not receive the submitted answer");
|
||||
for (const kind of ["MCP", "MEMORY"]) {
|
||||
const turn = host.startTurn({ text: `NATIVE_${kind}_FIXTURE: Use the native tool.`, requestId: `${kind.toLowerCase()}-turn` });
|
||||
for await (const _event of turn.events) { /* drain */ }
|
||||
assert.equal((await turn.result).status, "completed");
|
||||
}
|
||||
assert.deepEqual(bridgeCalls, [{ tool: "fixture_probe", arguments: { marker: "assigned-tool-roundtrip" } }]);
|
||||
assert.equal(host.steeringCapability()?.steering, true);
|
||||
assert.equal(host.steeringCapability()?.queuedFollowUp, false);
|
||||
const steering = host.startTurn({ text: "NATIVE_STEER_FIXTURE: Continue working until redirected.", requestId: "steering-turn" });
|
||||
let steered = false;
|
||||
for await (const event of steering.events) {
|
||||
if (!steered && event.type === "text_delta") {
|
||||
await host.steerActiveTurn("NATIVE_STEER_ACCEPTED: Answer with the transport marker instead.");
|
||||
steered = true;
|
||||
}
|
||||
}
|
||||
assert.equal(steered, true);
|
||||
assert.equal((await steering.result).status, "completed");
|
||||
assert.ok(requests.some(row => JSON.stringify(row.body.messages ?? []).includes("NATIVE_STEER_ACCEPTED")), "Accepted steering did not reach the native turn");
|
||||
await assert.rejects(host.steerActiveTurn("stale correction"), /active turn/);
|
||||
const stop = host.startTurn({ text: "NATIVE_STOP_FIXTURE: Continue working until cancelled.", requestId: "stop-turn" });
|
||||
let stopped = false;
|
||||
for await (const event of stop.events) {
|
||||
if (!stopped && event.type === "text_delta") {
|
||||
stopped = true;
|
||||
await host.interruptActiveTurn("qualification stop");
|
||||
}
|
||||
}
|
||||
assert.equal(stopped, true);
|
||||
assert.equal((await stop.result).status, "cancelled");
|
||||
const stoppedIdentity = acpxProviderSessionIdentity(host.identity(), host.binding());
|
||||
await host.close({ reason: "restore cancelled turn" });
|
||||
host = await AcpxRuntimeHost.open({ ...options, expectedIdentity: stoppedIdentity }, dependencies);
|
||||
const afterStop = host.startTurn({ text: "Reply with a fresh transport marker after cancellation.", requestId: "after-stop-turn" });
|
||||
const afterStopEvents = [];
|
||||
for await (const event of afterStop.events) afterStopEvents.push(event);
|
||||
const afterStopResult = await afterStop.result;
|
||||
assert.equal(afterStopResult.status, "completed", JSON.stringify(afterStopResult));
|
||||
assert.equal(afterStopEvents.filter(event => event.type === "text_delta" && event.stream !== "thought").map(event => event.text).join(""), "Hermes transport works.", "Cancelled history or old tool results polluted the restored turn");
|
||||
assert.ok(!afterStopEvents.some(event => event.type === "tool_call" || event.type === "tool_call_update"), "Old tools replayed as active work");
|
||||
assert.ok(requests.at(-1).body.messages.some(message => message.role === "user" && JSON.stringify(message.content).includes("NATIVE_STOP_FIXTURE")), "Cancelled turn history was discarded instead of restored");
|
||||
const restoreIdentity = acpxProviderSessionIdentity(host.identity(), host.binding());
|
||||
await host.close({ reason: "verify missing history" }); host = undefined;
|
||||
assert.match(await readFile(join(agentFiles, "hermes/memories/MEMORY.md"), "utf8"), /NATIVE_PERSISTED_MEMORY/);
|
||||
const nativeDatabases = (await readdir(join(root, "runtime"), { recursive: true })).filter(path => path.endsWith("state.db"));
|
||||
assert.equal(nativeDatabases.length, 1);
|
||||
await rm(join(root, "runtime", nativeDatabases[0]));
|
||||
let refused = false;
|
||||
try {
|
||||
host = await AcpxRuntimeHost.open({ ...options, expectedIdentity: restoreIdentity }, dependencies);
|
||||
const missing = host.startTurn({ text: "Continue the lost conversation.", requestId: "missing-history" });
|
||||
for await (const _event of missing.events) { /* drain */ }
|
||||
refused = (await missing.result).status !== "completed";
|
||||
} catch (error) {
|
||||
assert.match(String(error), /restore|history|session|resume/i);
|
||||
refused = true;
|
||||
}
|
||||
assert.equal(refused, true, "Missing native history was silently replaced with a new conversation");
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mkdtemp, mkdir, readFile, readdir, rm, symlink, link, writeFile } from "node:fs/promises";
|
||||
import { mkdtemp, mkdir, readFile, readdir, rename, rm, symlink, link, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
@@ -46,6 +46,22 @@ describe("Hermes managed learned state", () => {
|
||||
const f = await fixture();
|
||||
await expect(stageHermesAgentState(f.home, null)).rejects.toThrow(/managed agent-file storage/);
|
||||
});
|
||||
it.each(["symlink", "replacement"])("rejects a %s state parent without changing the redirected files", async kind => {
|
||||
const f = await fixture();
|
||||
const state = await stageHermesAgentState(f.home, f.context);
|
||||
const outside = join(f.root, "outside");
|
||||
await mkdir(outside); await mkdir(join(outside, "memories")); await mkdir(join(outside, "skills"));
|
||||
await writeFile(join(outside, "memories/MEMORY.md"), "Outside memory");
|
||||
await writeFile(join(outside, "memories/keep.md"), "Must not be deleted");
|
||||
await writeFile(join(f.home, "memories/MEMORY.md"), "Native change");
|
||||
await rename(join(f.agent, "hermes"), join(f.agent, "original-hermes"));
|
||||
if (kind === "symlink") await symlink(outside, join(f.agent, "hermes"));
|
||||
else await rename(outside, join(f.agent, "hermes"));
|
||||
await expect(state.collect()).rejects.toThrow(/unsafe directory|identity changed/);
|
||||
const redirected = kind === "symlink" ? outside : join(f.agent, "hermes");
|
||||
expect(await readFile(join(redirected, "memories/MEMORY.md"), "utf8")).toBe("Outside memory");
|
||||
expect(await readFile(join(redirected, "memories/keep.md"), "utf8")).toBe("Must not be deleted");
|
||||
});
|
||||
it("discards unfinished transfers during staging and collection without learning their contents", async () => {
|
||||
const f = await fixture();
|
||||
const state = await stageHermesAgentState(f.home, f.context);
|
||||
|
||||
@@ -18,13 +18,19 @@ export async function stageHermesAgentState(home: string, context: NativeRuntime
|
||||
await assertDirectory(agentRoot);
|
||||
const stateRoot = join(agentRoot, "hermes");
|
||||
await ensureDirectory(stateRoot);
|
||||
for (const name of DIRECTORIES) await mirrorDirectory(join(stateRoot, name), join(home, name));
|
||||
const agentIdentity = await directoryIdentity(agentRoot);
|
||||
const stateIdentity = await directoryIdentity(stateRoot);
|
||||
const assertRoots = async () => {
|
||||
await assertIdentity(agentRoot, agentIdentity);
|
||||
await assertIdentity(stateRoot, stateIdentity);
|
||||
};
|
||||
for (const name of DIRECTORIES) await mirrorDirectory(join(stateRoot, name), join(home, name), assertRoots);
|
||||
return {
|
||||
async collect() {
|
||||
// Called after verified process exit and before the controller collects
|
||||
// the existing working copy. Its usual authorization and save receipt apply.
|
||||
await assertDirectory(agentRoot);
|
||||
for (const name of DIRECTORIES) await mirrorDirectory(join(home, name), join(stateRoot, name));
|
||||
await assertRoots();
|
||||
for (const name of DIRECTORIES) await mirrorDirectory(join(home, name), join(stateRoot, name), assertRoots);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -33,6 +39,15 @@ async function assertDirectory(path: string) {
|
||||
const stat = await lstat(path);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Hermes state contains an unsafe directory");
|
||||
}
|
||||
async function directoryIdentity(path: string) {
|
||||
await assertDirectory(path);
|
||||
const stat = await lstat(path, { bigint: true });
|
||||
return { dev: stat.dev, ino: stat.ino };
|
||||
}
|
||||
async function assertIdentity(path: string, expected: { dev: bigint; ino: bigint }) {
|
||||
const actual = await directoryIdentity(path);
|
||||
if (actual.dev !== expected.dev || actual.ino !== expected.ino) throw new Error("Hermes state directory identity changed");
|
||||
}
|
||||
async function ensureDirectory(path: string) {
|
||||
await mkdir(path, { mode: 0o700 }).catch(error => { if (error.code !== "EEXIST") throw error; });
|
||||
await assertDirectory(path);
|
||||
@@ -64,21 +79,41 @@ async function inventory(root: string): Promise<Map<string, number>> {
|
||||
await walk(root);
|
||||
return files;
|
||||
}
|
||||
async function mirrorDirectory(source: string, destination: string) {
|
||||
async function mirrorDirectory(source: string, destination: string, assertRoots: () => Promise<void>) {
|
||||
await assertRoots();
|
||||
await ensureDirectory(source);
|
||||
await ensureDirectory(destination);
|
||||
const files = await inventory(source);
|
||||
const previous = await inventory(destination);
|
||||
const sourceIdentity = await directoryIdentity(source);
|
||||
const destinationIdentity = await directoryIdentity(destination);
|
||||
const assertBoundRoots = async () => {
|
||||
await assertRoots();
|
||||
await assertIdentity(source, sourceIdentity);
|
||||
await assertIdentity(destination, destinationIdentity);
|
||||
};
|
||||
for (const [name, size] of files) {
|
||||
await assertBoundRoots();
|
||||
const target = join(destination, name);
|
||||
const parts = relative(destination, dirname(target)).split(/[\\/]/).filter(Boolean);
|
||||
let directory = destination;
|
||||
for (const part of parts) { directory = join(directory, part); await ensureDirectory(directory); }
|
||||
const parents: { path: string; identity: { dev: bigint; ino: bigint } }[] = [];
|
||||
for (const part of parts) {
|
||||
await assertBoundRoots();
|
||||
for (const parent of parents) await assertIdentity(parent.path, parent.identity);
|
||||
directory = join(directory, part); await ensureDirectory(directory);
|
||||
parents.push({ path: directory, identity: await directoryIdentity(directory) });
|
||||
}
|
||||
const assertDestination = async () => {
|
||||
await assertBoundRoots();
|
||||
for (const parent of parents) await assertIdentity(parent.path, parent.identity);
|
||||
};
|
||||
const input = await open(join(source, name), constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const temporary = join(dirname(target), `.paperclip-hermes-transfer-${randomUUID()}.tmp`);
|
||||
try {
|
||||
const before = await input.stat({ bigint: true });
|
||||
if (!before.isFile() || before.nlink !== 1n || before.size !== BigInt(size)) throw new Error("Hermes state changed during collection");
|
||||
await assertDestination();
|
||||
const output = await open(temporary, "wx", 0o600);
|
||||
try {
|
||||
const buffer = Buffer.alloc(Math.min(size + 1, 64 * 1024));
|
||||
@@ -93,8 +128,20 @@ async function mirrorDirectory(source: string, destination: string) {
|
||||
} finally { await output.close(); }
|
||||
const after = await input.stat({ bigint: true });
|
||||
if (before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs || before.size !== after.size) throw new Error("Hermes state changed during collection");
|
||||
await assertDestination();
|
||||
await rename(temporary, target);
|
||||
} finally { await input.close(); await rm(temporary, { force: true }); }
|
||||
} finally {
|
||||
await input.close();
|
||||
await assertDestination();
|
||||
await rm(temporary, { force: true });
|
||||
}
|
||||
}
|
||||
for (const name of previous.keys()) if (!files.has(name)) {
|
||||
await assertBoundRoots();
|
||||
let directory = destination;
|
||||
for (const part of relative(destination, dirname(join(destination, name))).split(/[\\/]/).filter(Boolean)) {
|
||||
directory = join(directory, part); await assertDirectory(directory);
|
||||
}
|
||||
await rm(join(destination, name));
|
||||
}
|
||||
for (const name of previous.keys()) if (!files.has(name)) await rm(join(destination, name));
|
||||
}
|
||||
@@ -1461,17 +1461,14 @@ immutable Daytona image; no private hooks or fixture database writes are used.
|
||||
|
||||
## Extended ACP harnesses (explicit only)
|
||||
|
||||
`--suite extended-harnesses` declares 40 Product E2E cells: Cursor, Copilot,
|
||||
Pi and Hermes on local and Daytona, each exercising authenticated completion,
|
||||
`--suite extended-harnesses` declares 30 Product E2E cells: Cursor, Copilot,
|
||||
and Pi on local and Daytona, each exercising authenticated completion,
|
||||
question/answer continuation, revision-bound semantic plan approval, restart
|
||||
with pending input, and file edit plus independent byte validation. The file
|
||||
case uses a public project workspace so Daytona copy-back is graded too.
|
||||
These are candidate definitions, not a claim of provider qualification. Native
|
||||
provider-specific questions, plan decisions, restrictive permissions and steering
|
||||
need their separate conformance/qualification evidence.
|
||||
Hermes uses an existing managed OpenRouter connection through the ordinary
|
||||
connection fixture. It remains pending qualification; the native transport
|
||||
fixture does not substitute for these browser and remote workflows.
|
||||
|
||||
```sh
|
||||
pnpm test:e2e:runner -- --list --suite extended-harnesses
|
||||
|
||||
@@ -146,10 +146,10 @@ describe("runner E2E catalog", () => {
|
||||
expect(localIntegrityTasks).toHaveLength(2);
|
||||
expect(openRouterBreadthTasks).toHaveLength(3);
|
||||
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
|
||||
63, 12, 6, 8, 2, 2, 2, 40, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116,
|
||||
63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116,
|
||||
]);
|
||||
expect(validateRunnerCatalog()).toHaveLength(708);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(708);
|
||||
expect(validateRunnerCatalog()).toHaveLength(698);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(698);
|
||||
expect(
|
||||
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
|
||||
).toHaveLength(48);
|
||||
|
||||
@@ -174,7 +174,7 @@ function nativeProfile(input: {
|
||||
provider: "codex" | "opencode" | "acpx";
|
||||
model: string;
|
||||
credential: RunnerProfileFixture["credential"];
|
||||
acpxAgent?: "claude" | "codex" | "grok" | "cursor" | "copilot" | "pi" | "hermes";
|
||||
acpxAgent?: "claude" | "codex" | "grok" | "cursor" | "copilot" | "pi";
|
||||
qualificationCandidate?: RunnerProfileFixture["qualificationCandidate"];
|
||||
supportedEnvironments?: readonly (typeof ENVIRONMENT_IDS)[number][];
|
||||
modelQualification?: RunnerProfileFixture["modelQualification"];
|
||||
@@ -338,11 +338,6 @@ export const runnerProfiles: readonly RunnerProfileFixture[] = [
|
||||
// Explicit qualification choices from authenticated model discovery, not shipped
|
||||
// defaults. Cursor has local/Daytona proof; Copilot and Pi remain pending.
|
||||
export const extendedHarnessProfiles: readonly RunnerProfileFixture[] = [
|
||||
nativeProfile({
|
||||
id: "runner-acpx-hermes", label: "Runner Hermes (candidate)", provider: "acpx", acpxAgent: "hermes",
|
||||
qualificationCandidate: "hermes", credential: "OPENROUTER_API_KEY", model: "deepseek/deepseek-v4-flash-0731",
|
||||
modelQualification: { source: "candidate_runner_profile", qualificationId: "hermes:v2026.9.24:openrouter:pending" },
|
||||
}),
|
||||
nativeProfile({
|
||||
id: "runner-acpx-cursor", label: "Runner Cursor", provider: "acpx", acpxAgent: "cursor",
|
||||
qualificationCandidate: "cursor", credential: "CURSOR_AUTH_TOKEN",
|
||||
@@ -1185,7 +1180,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
description: "Explicit candidate qualification through real Paperclip tools, browser interactions, file edits and restart recovery.",
|
||||
groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments,
|
||||
tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask],
|
||||
expectedMatrixSize: 40,
|
||||
expectedMatrixSize: 30,
|
||||
definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28" },
|
||||
},
|
||||
{
|
||||
|
||||
@@ -7,7 +7,7 @@ import { findSecretLeak, redactText } from "./redaction.js";
|
||||
const selected = runnerMatrix.filter(cell => cell.suite.id === "extended-harnesses");
|
||||
describe("extended ACP harness qualification", () => {
|
||||
it("has exactly five real product journeys per provider in both environments, excluded from all", () => {
|
||||
expect(selected).toHaveLength(40);
|
||||
expect(selected).toHaveLength(30);
|
||||
expect(new Set(selected.map(cell => cell.environment.id))).toEqual(new Set(["local", "daytona"]));
|
||||
expect(new Set(selected.map(cell => cell.task.id))).toEqual(new Set([
|
||||
"hello-complete", "question-resume-complete", "plan-approve-complete", "structured-question-restart-resume", "file-edit-validate",
|
||||
@@ -18,7 +18,6 @@ describe("extended ACP harness qualification", () => {
|
||||
});
|
||||
it("uses exact discovered models, encrypted credential references and current qualification metadata", () => {
|
||||
expect(extendedHarnessProfiles.map(profile => profile.model)).toEqual([
|
||||
"deepseek/deepseek-v4-flash-0731",
|
||||
"gpt-5.6-luna[context=272k,reasoning=medium,fast=false]", "gpt-5.6-luna", "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
]);
|
||||
for (const profile of extendedHarnessProfiles) {
|
||||
@@ -29,7 +28,7 @@ describe("extended ACP harness qualification", () => {
|
||||
}
|
||||
});
|
||||
it("replaces ambient admission with the selected exact pairs and strips raw provider credentials from server", () => {
|
||||
const cell = selected.find(cell => cell.profile.qualificationCandidate === "cursor")!;
|
||||
const cell = selected[0]!;
|
||||
const ambient = { PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient", CURSOR_AUTH_TOKEN: "cursor", CURSOR_API_KEY: "cursor-key", COPILOT_GITHUB_TOKEN: "copilot", GH_TOKEN: "github", GITHUB_TOKEN: "github" };
|
||||
const env = buildRunnerE2EProcessEnvironment(ambient, [cell]);
|
||||
expect(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
|
||||
|
||||
@@ -92,7 +92,6 @@ describe("live runner fixtures", () => {
|
||||
});
|
||||
|
||||
it.each([
|
||||
["runner-acpx-hermes", "extended-harnesses", "hello-complete"],
|
||||
["runner-codex", "hiring-templates", "hire-coder-template-reuse"],
|
||||
["runner-acpx-claude", "hiring-templates", "hire-coder-template-reuse"],
|
||||
["runner-codex", "everyday-workflows", "hire-reuse"],
|
||||
@@ -111,7 +110,7 @@ describe("live runner fixtures", () => {
|
||||
e.environment.id === "local",
|
||||
)!;
|
||||
const provider =
|
||||
profile === "runner-acpx-claude" ? "anthropic" : ["runner-opencode", "runner-acpx-hermes"].includes(profile) ? "openrouter" : "openai";
|
||||
profile === "runner-acpx-claude" ? "anthropic" : profile === "runner-opencode" ? "openrouter" : "openai";
|
||||
let connected = false;
|
||||
let agentBody: any;
|
||||
const api = {
|
||||
|
||||
@@ -241,8 +241,7 @@ export async function setupLiveFixtures(input: {
|
||||
},
|
||||
});
|
||||
|
||||
const managedHiring = isManagedHiringCase(execution.suite.id, execution.task.id)
|
||||
|| execution.profile.qualificationCandidate === "hermes";
|
||||
const managedHiring = isManagedHiringCase(execution.suite.id, execution.task.id);
|
||||
if (managedHiring) {
|
||||
registry.register<ManagedAccountFixture>({
|
||||
id: "ai-connection",
|
||||
|
||||
@@ -78,7 +78,7 @@ export interface RunnerProfileFixture {
|
||||
| "openrouter_rankings_snapshot";
|
||||
qualificationId: string;
|
||||
};
|
||||
qualificationCandidate?: "cursor" | "copilot" | "pi" | "hermes";
|
||||
qualificationCandidate?: "cursor" | "copilot" | "pi";
|
||||
ranking?: {
|
||||
rank: number;
|
||||
canonicalModelId: string;
|
||||
|
||||
Reference in new issue
Block a user