mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-29 04:25:52 +01:00
211 lines
17 KiB
TypeScript
211 lines
17 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { readFileSync } from "node:fs";
|
|
import { isDeepStrictEqual } from "node:util";
|
|
import type { AiProviderRouting } from "../../packages/shared/src/ai-provider-routing.js";
|
|
import { hasAcpxNativeOrigin } from "./acpx-native-origin.js";
|
|
import type { RunnerProfileFixture, RunnerTaskFixture } from "./types.js";
|
|
|
|
const record = (value: unknown): Record<string, unknown> =>
|
|
value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
const present = (value: unknown) => typeof value === "string" && value.trim().length > 0;
|
|
export const HERMES_API_CONNECTION_BUDGET_CENTS = 200;
|
|
export const HERMES_NATIVE_INTERACTION_SUITE = "hermes-native-interactions";
|
|
export const isHermesConnectionSuite = (suiteId: string) =>
|
|
suiteId === "hermes-api-connections" || suiteId === "hermes-bedrock-connections" || suiteId === HERMES_NATIVE_INTERACTION_SUITE;
|
|
export const isHermesOpenRouterWorkflow = (execution: {
|
|
suite: { id: string }; profile: { qualificationCandidate?: string; credential: string };
|
|
}) => execution.suite.id === "extended-harnesses"
|
|
&& execution.profile.qualificationCandidate === "hermes" && execution.profile.credential === "OPENROUTER_API_KEY";
|
|
export const hermesBedrockConnectionChoice = {
|
|
credential: "AWS_BEARER_TOKEN_BEDROCK",
|
|
model: "us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
|
routing: { kind: "bedrock", protocol: "bedrock", auth: "bearer", region: "us-east-1",
|
|
models: [{ id: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }] },
|
|
} as const satisfies { credential: RunnerProfileFixture["credential"]; model: string; routing: AiProviderRouting };
|
|
|
|
/** The answer is generated by the reviewer and is absent from the model prompt. */
|
|
export const hermesNativeAnswerText = (nonce: string) =>
|
|
`constraint-${createHash("sha256").update(`hermes-native-answer:${nonce}`).digest("hex").slice(0, 20)}`;
|
|
|
|
export const hermesNativeQuestionTask: RunnerTaskFixture = {
|
|
id: "native-question-batch-reconnect", label: "Native question batch survives browser reconnect",
|
|
groups: ["native"], workMode: "standard", flow: "native_question_completion", expectedRunCount: 1,
|
|
automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 300_000, daytona: 420_000 },
|
|
expectedTerminalState: { issue: "done", run: "succeeded" },
|
|
buildTitle: nonce => `Hermes native questions ${nonce}`,
|
|
buildPrompt: nonce => [
|
|
"Call Hermes's native clarify tool exactly once with a questions batch containing these three questions, in this order:",
|
|
JSON.stringify([
|
|
{ id: "color", question: "Choose the fixture color", choices: ["Cobalt", "Amber"] },
|
|
{ id: "targets", question: "Choose the fixture targets", choices: ["Linux", "Mac"], multi_select: true },
|
|
{ id: "notes", question: "Describe the fixture constraint" },
|
|
]),
|
|
"Wait for the actual native callback. Do not answer your own questions. Do not guess the reviewer's selections. Do not substitute request_human_input. Do not ask through prose. Do not create files or plans. If clarify is unavailable, report that limitation without claiming completion.",
|
|
`After the actual native result arrives, form your summary as HERMES-NATIVE-${nonce}-<exact returned notes user_response>. The notes are supplied by the reviewer; copy that returned text verbatim.`,
|
|
"Use the current turn completion revision and criterion IDs. Call paperclip_finish exactly once with {reportedWorkDisposition:'done',summary:COMPUTED_SUMMARY,completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:[{criterionId:'objective',status:'satisfied',evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. COMPUTED_SUMMARY and CURRENT_TURN_REVISION are placeholders for the actual returned-text summary and current revision. Wait for acceptance, then emit only that exact summary as your final answer.",
|
|
].join("\n"),
|
|
buildVisibleMarker: nonce => `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}`,
|
|
buildMatchers: (nonce, execution) => [
|
|
{ kind: "message_exact", expected: `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}` },
|
|
{ kind: "message_occurrences", expected: `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}`, count: 1 },
|
|
{ kind: "issue_status", expected: "done" }, { kind: "run_status", expected: "succeeded" },
|
|
{ kind: "runtime_mode", expected: "native" }, { kind: "environment", expected: execution.environment.id },
|
|
],
|
|
};
|
|
|
|
/** Preserve the pinned native form; semantic tools cannot satisfy this oracle. */
|
|
export function hasHermesNativeQuestionBatch(value: unknown): boolean {
|
|
const set = record(value), questions = Array.isArray(set.questions) ? set.questions.map(record) : [];
|
|
const expected = [
|
|
["q0", "Choose the fixture color", "single_select", ["Cobalt", "Amber"]],
|
|
["q1", "Choose the fixture targets", "multi_select", ["Linux", "Mac"]],
|
|
["q2", "Describe the fixture constraint", "text", []],
|
|
] as const;
|
|
return set.schema === "paperclip.question_set.v1" && questions.length === expected.length
|
|
&& questions.every((question, index) => {
|
|
const [id, prompt, mode, labels] = expected[index]!;
|
|
const options = Array.isArray(question.options) ? question.options.map(record) : [];
|
|
return question.id === id && question.prompt === prompt && question.answerMode === mode && question.required === true
|
|
&& options.length === labels.length && options.every((option, optionIndex) => option.id === `o${optionIndex}`
|
|
&& [labels[optionIndex], `${labels[optionIndex]} (Recommended)`].includes(String(option.label)))
|
|
&& (labels.length === 0 || record(question.customAnswer).enabled === true);
|
|
});
|
|
}
|
|
|
|
/** Require one exact ordered, post-write delivery receipt for this native callback. */
|
|
export function hasExactHermesNativeQuestionResponse(input: {
|
|
events: readonly unknown[]; runId: string; turnId: string; requestId: string; questionSet: unknown; response: unknown;
|
|
}): boolean {
|
|
if (![input.runId, input.turnId, input.requestId].every(present) || !hasHermesNativeQuestionBatch(input.questionSet)
|
|
|| record(input.response).schema !== "paperclip.question_response.v1") return false;
|
|
const rows = input.events.map(record).map(row => ({ row, event: record(record(row.payload).prpEvent) }));
|
|
if (rows.filter(({ row, event }) => row.runId === input.runId && event.eventType === "runtime_request.created"
|
|
&& hasAcpxNativeOrigin(record(record(event.payload).request).origin, "hermes", "_hermes/ask_questions")).length !== 1) return false;
|
|
const relevant = rows
|
|
.filter(({ event }) => event.eventType === "runtime_request.created"
|
|
? record(record(event.payload).request).requestId === input.requestId
|
|
: ["runtime_request.resolved", "runtime_request.expired", "runtime_request.cancelled"].includes(String(event.eventType))
|
|
&& record(event.payload).requestId === input.requestId);
|
|
if (relevant.some(({ row, event }) => row.runId !== input.runId || event.runId !== input.runId
|
|
|| event.turnId !== input.turnId || event.schema !== "paperclip.prp.event.v1"
|
|
|| event.schemaVersion !== 1 || row.protocolSchemaVersion !== 1)) return false;
|
|
const created = relevant.filter(({ event }) => event.eventType === "runtime_request.created");
|
|
const outcomes = relevant.filter(({ event }) => event.eventType !== "runtime_request.created");
|
|
if (created.length !== 1 || outcomes.length !== 1) return false;
|
|
const request = record(record(created[0]!.event.payload).request), outcome = outcomes[0]!.event;
|
|
return hasAcpxNativeOrigin(request.origin, "hermes", "_hermes/ask_questions")
|
|
&& request.type === "input" && request.status === "pending" && request.turnId === input.turnId
|
|
&& isDeepStrictEqual(request.input, input.questionSet)
|
|
&& outcome.eventType === "runtime_request.resolved" && record(outcome.payload).action === "submit"
|
|
&& record(outcome.payload).turnId === input.turnId && isDeepStrictEqual(record(outcome.payload).response, input.response)
|
|
&& Number.isSafeInteger(created[0]!.event.sourceSeq) && Number.isSafeInteger(outcome.sourceSeq)
|
|
&& (created[0]!.event.sourceSeq as number) >= 0 && (outcome.sourceSeq as number) > (created[0]!.event.sourceSeq as number);
|
|
}
|
|
|
|
/** Establish the intended personal-account user independently, before a paid task. */
|
|
export async function captureHermesApiAccountOwner(input: {
|
|
api: { get<T>(path: string): Promise<T> }; companyId: string; connectionId: string; provider: string;
|
|
expectedRouting?: AiProviderRouting; expectedGrantId?: string;
|
|
}) {
|
|
const listed = await input.api.get<{ currentUserId: string; connections: {
|
|
id: string; companyId: string; provider: string; method: string; ownership: string; ownerUserId?: string; status: string;
|
|
routing?: AiProviderRouting; grantId?: string;
|
|
}[] }>(`/api/companies/${input.companyId}/ai-connections`);
|
|
const matches = listed.connections.filter(connection => connection.id === input.connectionId);
|
|
const selected = matches[0];
|
|
return { expectedResponsibleUserId: selected?.ownerUserId ?? null,
|
|
...(input.expectedRouting ? { routing: { expected: input.expectedRouting, observed: selected?.routing ?? null } } : {}),
|
|
checks: [
|
|
{ id: "selected-account-public-readback", passed: matches.length === 1 && selected?.companyId === input.companyId && selected.provider === input.provider && selected.method === "api_key" && selected.ownership === "personal" && selected.status === "connected" },
|
|
{ id: "authenticated-caller-known", passed: present(listed.currentUserId) },
|
|
{ id: "selected-account-owned-by-caller", passed: present(selected?.ownerUserId) && selected?.ownerUserId === listed.currentUserId },
|
|
...(input.expectedRouting ? [{ id: "selected-account-routing", passed: isDeepStrictEqual(selected?.routing, input.expectedRouting) }] : []),
|
|
...(input.expectedGrantId !== undefined ? [{ id: "selected-account-grant", passed: present(input.expectedGrantId) && selected?.grantId === input.expectedGrantId }] : []),
|
|
] };
|
|
}
|
|
|
|
/** Read both public budgets before creating the paid task. Unknown usage stays unknown. */
|
|
export async function captureHermesApiBudgets(input: {
|
|
api: { get<T>(path: string): Promise<T> }; companyId: string; agentId: string;
|
|
}) {
|
|
const [company, agent] = await Promise.all([
|
|
input.api.get<{ id: string; budgetMonthlyCents: unknown }>(`/api/companies/${input.companyId}`),
|
|
input.api.get<{ id: string; companyId: string; budgetMonthlyCents: unknown }>(`/api/agents/${input.agentId}`),
|
|
]);
|
|
return { budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, checks: [
|
|
{ id: "company-budget", passed: company.id === input.companyId && company.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS },
|
|
{ id: "agent-budget", passed: agent.id === input.agentId && agent.companyId === input.companyId && agent.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS },
|
|
] };
|
|
}
|
|
|
|
/** Observe settled billing and budget health before fixture cleanup pauses the agent. */
|
|
export async function captureHermesOpenRouterSettlement(input: {
|
|
api: { get<T>(path: string): Promise<T> }; companyId: string; agentId: string; issueId: string; runId: string;
|
|
}) {
|
|
const [company, agent, run] = await Promise.all([
|
|
input.api.get<Record<string, unknown>>(`/api/companies/${input.companyId}`),
|
|
input.api.get<Record<string, unknown>>(`/api/agents/${input.agentId}`),
|
|
input.api.get<Record<string, unknown>>(`/api/heartbeat-runs/${input.runId}`),
|
|
]);
|
|
const usage = record(run.usageJson), provenance = record(usage.pricingProvenance);
|
|
const cost = usage.costUsd, exact = usage.costUsdExact;
|
|
return { observation: {
|
|
company: { id: company.id, status: company.status, pauseReason: company.pauseReason, budgetMonthlyCents: company.budgetMonthlyCents },
|
|
agent: { id: agent.id, companyId: agent.companyId, status: agent.status, pauseReason: agent.pauseReason, budgetMonthlyCents: agent.budgetMonthlyCents },
|
|
run: { id: run.id, companyId: run.companyId, agentId: run.agentId, issueId: run.issueId, status: run.status,
|
|
costAccountingPending: run.costAccountingPending, costAccountedAt: run.costAccountedAt,
|
|
usage: Object.fromEntries(["provider", "biller", "billingType", "costStatus", "costUsd", "costUsdExact", "inputTokens", "outputTokens", "accountingReceiptReady", "pricingProvenance"].map(key => [key, usage[key]])) },
|
|
}, checks: [
|
|
{ id: "billing-observation-scope", passed: company.id === input.companyId && agent.id === input.agentId && agent.companyId === input.companyId
|
|
&& run.id === input.runId && run.companyId === input.companyId && run.agentId === input.agentId && run.issueId === input.issueId },
|
|
{ id: "settled-openrouter-reported-cost", passed: run.status === "succeeded" && run.costAccountingPending === false && present(run.costAccountedAt)
|
|
&& usage.accountingReceiptReady === true && usage.biller === "openrouter" && usage.billingType === "metered_api" && usage.costStatus === "reported"
|
|
&& provenance.source === "provider_reported" && provenance.version === "hermes-openrouter-wire/v1"
|
|
&& typeof cost === "number" && Number.isFinite(cost) && cost >= 0 && typeof exact === "string" && /^(0|[1-9][0-9]{0,6})\.[0-9]{9}$/.test(exact) && Number(exact) === cost
|
|
&& typeof usage.inputTokens === "number" && Number.isSafeInteger(usage.inputTokens) && usage.inputTokens >= 0
|
|
&& typeof usage.outputTokens === "number" && Number.isSafeInteger(usage.outputTokens) && usage.outputTokens >= 0 && usage.inputTokens + usage.outputTokens > 0 },
|
|
{ id: "budget-health-after-settlement", passed: company.status === "active" && agent.status === "idle" && agent.pauseReason === null
|
|
&& company.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS && agent.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS },
|
|
] };
|
|
}
|
|
|
|
/** Grade public run/account/model metadata; a model's completion claim cannot supply it. */
|
|
export function gradeHermesApiConnection(input: {
|
|
companyId: string; agentId: string; issueId: string; connectionId: string; provider: string; model: string; expectedResponsibleUserId: string;
|
|
accountMode?: "responsible_user" | "delegated"; expectedGrantId?: string;
|
|
runs: readonly {
|
|
companyId: string; agentId: string; status: string; runtimeMode?: string;
|
|
issueId?: string | null; responsibleUserId?: string | null;
|
|
contextSnapshot?: Record<string, unknown> | null; runnerProfileJson?: Record<string, unknown> | null;
|
|
}[];
|
|
}) {
|
|
const run = input.runs[0];
|
|
const context = record(run?.contextSnapshot), account = record(context.aiConnection);
|
|
const identity = record(record(record(run?.runnerProfileJson).sessionCheckpoint).providerIdentity);
|
|
const provider = record(record(record(run?.runnerProfileJson).nativeExecutionInput).provider);
|
|
return [
|
|
{ id: "one-successful-native-run", passed: input.runs.length === 1 && run?.status === "succeeded" && run.runtimeMode === "native" },
|
|
{ id: "company-agent-task-scope", passed: run?.companyId === input.companyId && run.agentId === input.agentId && run.issueId === input.issueId },
|
|
{ id: "selected-managed-api-account", passed: account.connectionId === input.connectionId && account.provider === input.provider && account.method === "api_key" && account.mode === (input.accountMode ?? "responsible_user") },
|
|
...(input.expectedGrantId !== undefined ? [{ id: "selected-managed-account-grant", passed: present(input.expectedGrantId) && account.grantId === input.expectedGrantId }] : []),
|
|
{ id: "responsible-user-attribution", passed: present(input.expectedResponsibleUserId) && run?.responsibleUserId === input.expectedResponsibleUserId && account.responsibleUserId === input.expectedResponsibleUserId },
|
|
{ id: "native-hermes-provider", passed: provider.kind === "acpx" && provider.agent === "hermes" && provider.model === input.model },
|
|
{ id: "exact-native-model", passed: identity.kind === "acpx" && identity.requestedModel === input.model && identity.effectiveModel === input.model },
|
|
];
|
|
}
|
|
|
|
/** Explicit authenticated catalog choices; none is a production default or live qualification. */
|
|
export const hermesApiConnectionChoices = [
|
|
{ provider: "anthropic", credential: "ANTHROPIC_API_KEY", model: "claude-haiku-4-5-20251001" },
|
|
{ provider: "openai", credential: "OPENAI_API_KEY", model: "gpt-5.6-luna" },
|
|
{ provider: "xai", credential: "XAI_API_KEY", model: "grok-4.7" },
|
|
// Google restricts 2.5 models to prior users even when they appear in the catalog.
|
|
// https://ai.google.dev/gemini-api/docs/deprecations
|
|
{ provider: "google", credential: "GEMINI_API_KEY", model: "gemini-3.8-flash" },
|
|
] as const satisfies readonly { provider: string; credential: RunnerProfileFixture["credential"]; model: string }[];
|
|
|
|
export const hermesApiConnectionDefinitionDigest = createHash("sha256").update(
|
|
["hermes-api-connections.ts", "acpx-native-origin.ts", "live-fixtures.ts", "harness-env.ts", "runner.spec.ts"]
|
|
.map(file => readFileSync(new URL(`./${file}`, import.meta.url), "utf8")).join("\n"),
|
|
).digest("hex");
|