From d5dd226722d0bc8d712f8453d8a50f50880b9e11 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 29 Apr 2026 11:13:33 -0500 Subject: [PATCH] Apply runtime model profiles to heartbeat config Co-Authored-By: Paperclip --- .../__tests__/heartbeat-model-profile.test.ts | 123 ++++++++++ server/src/adapters/index.ts | 1 + server/src/services/heartbeat.ts | 223 +++++++++++++++++- 3 files changed, 336 insertions(+), 11 deletions(-) create mode 100644 server/src/__tests__/heartbeat-model-profile.test.ts diff --git a/server/src/__tests__/heartbeat-model-profile.test.ts b/server/src/__tests__/heartbeat-model-profile.test.ts new file mode 100644 index 00000000000..726436caafe --- /dev/null +++ b/server/src/__tests__/heartbeat-model-profile.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import type { AdapterModelProfileDefinition } from "../adapters/index.js"; +import { + mergeModelProfileAdapterConfig, + normalizeModelProfileWakeContext, + resolveModelProfileApplication, +} from "../services/heartbeat.ts"; + +const cheapProfile: AdapterModelProfileDefinition = { + key: "cheap", + label: "Cheap", + adapterConfig: { + model: "adapter-cheap", + modelReasoningEffort: "low", + }, + source: "adapter_default", +}; + +describe("heartbeat model profile application", () => { + it("applies cheap profile patches before explicit issue adapter config overrides", () => { + const modelProfile = resolveModelProfileApplication({ + adapterModelProfiles: [cheapProfile], + agentRuntimeConfig: {}, + issueModelProfile: "cheap", + contextSnapshot: {}, + }); + + const merged = mergeModelProfileAdapterConfig({ + baseConfig: { + model: "primary", + modelReasoningEffort: "high", + approvalPolicy: "strict", + }, + modelProfile, + issueAdapterConfig: { + model: "issue-explicit", + }, + }); + + expect(modelProfile).toMatchObject({ + requested: "cheap", + requestedBy: "issue_override", + applied: "cheap", + configSource: "adapter_default", + fallbackReason: null, + }); + expect(merged).toEqual({ + model: "issue-explicit", + modelReasoningEffort: "low", + approvalPolicy: "strict", + }); + }); + + it("lets agent runtime profile config customize adapter defaults", () => { + const modelProfile = resolveModelProfileApplication({ + adapterModelProfiles: [cheapProfile], + agentRuntimeConfig: { + modelProfiles: { + cheap: { + adapterConfig: { + model: "agent-cheap", + }, + }, + }, + }, + issueModelProfile: null, + contextSnapshot: { modelProfile: "cheap" }, + }); + + expect(modelProfile).toMatchObject({ + requested: "cheap", + requestedBy: "wake_context", + applied: "cheap", + configSource: "agent_runtime", + adapterConfig: { + model: "agent-cheap", + modelReasoningEffort: "low", + }, + }); + }); + + it("falls back to the primary config when the adapter does not support the requested profile", () => { + const modelProfile = resolveModelProfileApplication({ + adapterModelProfiles: [], + agentRuntimeConfig: { + modelProfiles: { + cheap: { + adapterConfig: { + model: "agent-cheap", + }, + }, + }, + }, + issueModelProfile: null, + contextSnapshot: { modelProfile: "cheap" }, + }); + + const merged = mergeModelProfileAdapterConfig({ + baseConfig: { + model: "primary", + }, + modelProfile, + issueAdapterConfig: null, + }); + + expect(modelProfile).toMatchObject({ + requested: "cheap", + applied: null, + fallbackReason: "adapter_profile_not_supported", + adapterConfig: null, + }); + expect(merged).toEqual({ model: "primary" }); + }); + + it("normalizes a wake payload model profile into run context", () => { + const contextSnapshot = normalizeModelProfileWakeContext({ + contextSnapshot: {}, + payload: { modelProfile: "cheap" }, + }); + + expect(contextSnapshot).toMatchObject({ modelProfile: "cheap" }); + }); +}); diff --git a/server/src/adapters/index.ts b/server/src/adapters/index.ts index 5d193649ff8..0f713c9cf09 100644 --- a/server/src/adapters/index.ts +++ b/server/src/adapters/index.ts @@ -16,6 +16,7 @@ export type { AdapterExecutionContext, AdapterExecutionResult, AdapterInvocationMeta, + AdapterModelProfileDefinition, AdapterEnvironmentCheckLevel, AdapterEnvironmentCheck, AdapterEnvironmentTestStatus, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index a602bf1df33..a76b15a8145 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -40,8 +40,14 @@ import { conflict, HttpError, notFound } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { publishLiveEvent } from "./live-events.js"; import { getRunLogStore, type RunLogHandle } from "./run-log-store.js"; -import { getServerAdapter, runningProcesses } from "../adapters/index.js"; -import type { AdapterExecutionResult, AdapterInvocationMeta, AdapterSessionCodec, UsageSummary } from "../adapters/index.js"; +import { getServerAdapter, listAdapterModelProfiles, runningProcesses } from "../adapters/index.js"; +import type { + AdapterExecutionResult, + AdapterInvocationMeta, + AdapterModelProfileDefinition, + AdapterSessionCodec, + UsageSummary, +} from "../adapters/index.js"; import { createLocalAgentJwt } from "../agent-auth-jwt.js"; import { parseObject, asBoolean, asNumber, appendWithByteCap, MAX_EXCERPT_BYTES } from "../adapters/utils.js"; import { costService } from "./costs.js"; @@ -886,6 +892,18 @@ interface ParsedIssueAssigneeAdapterOverrides { useProjectWorkspace: boolean | null; } +type ModelProfileRequestSource = "issue_override" | "wake_context"; +type AppliedModelProfileConfigSource = "agent_runtime" | "adapter_default"; + +export interface ModelProfileApplication { + requested: ModelProfileKey | null; + requestedBy: ModelProfileRequestSource | null; + applied: ModelProfileKey | null; + configSource: AppliedModelProfileConfigSource | null; + fallbackReason: string | null; + adapterConfig: Record | null; +} + export type ResolvedWorkspaceForRun = { cwd: string; source: "project_primary" | "task_session" | "agent_home"; @@ -920,6 +938,147 @@ function readNonEmptyString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } +function readModelProfileKey(value: unknown): ModelProfileKey | null { + return MODEL_PROFILE_KEYS.includes(value as ModelProfileKey) + ? (value as ModelProfileKey) + : null; +} + +function readContextModelProfile( + contextSnapshot: Record | null | undefined, +): ModelProfileKey | null { + return readModelProfileKey(contextSnapshot?.modelProfile); +} + +export function normalizeModelProfileWakeContext(input: { + contextSnapshot: Record; + payload: Record | null | undefined; +}): Record { + const modelProfileFromPayload = readModelProfileKey(input.payload?.modelProfile); + if (!readContextModelProfile(input.contextSnapshot) && modelProfileFromPayload) { + input.contextSnapshot.modelProfile = modelProfileFromPayload; + } + return input.contextSnapshot; +} + +function readAgentRuntimeModelProfile( + runtimeConfig: unknown, + key: ModelProfileKey, +): { enabled: boolean; adapterConfig: Record; configured: boolean } { + const modelProfiles = parseObject(parseObject(runtimeConfig).modelProfiles); + const profile = parseObject(modelProfiles[key]); + if (Object.keys(profile).length === 0) { + return { enabled: true, adapterConfig: {}, configured: false }; + } + + return { + enabled: profile.enabled !== false, + adapterConfig: parseObject(profile.adapterConfig), + configured: true, + }; +} + +export function resolveModelProfileApplication(input: { + adapterModelProfiles: AdapterModelProfileDefinition[]; + agentRuntimeConfig: unknown; + issueModelProfile: ModelProfileKey | null | undefined; + contextSnapshot: Record | null | undefined; + profileResolutionFallbackReason?: string | null; +}): ModelProfileApplication { + const issueModelProfile = input.issueModelProfile ?? null; + const contextModelProfile = readContextModelProfile(input.contextSnapshot); + const requested = issueModelProfile ?? contextModelProfile; + const requestedBy: ModelProfileRequestSource | null = issueModelProfile + ? "issue_override" + : contextModelProfile + ? "wake_context" + : null; + + if (!requested) { + return { + requested: null, + requestedBy: null, + applied: null, + configSource: null, + fallbackReason: null, + adapterConfig: null, + }; + } + + const adapterProfile = input.adapterModelProfiles.find((profile) => profile.key === requested) ?? null; + if (!adapterProfile) { + return { + requested, + requestedBy, + applied: null, + configSource: null, + fallbackReason: input.profileResolutionFallbackReason ?? "adapter_profile_not_supported", + adapterConfig: null, + }; + } + + const runtimeProfile = readAgentRuntimeModelProfile(input.agentRuntimeConfig, requested); + if (!runtimeProfile.enabled) { + return { + requested, + requestedBy, + applied: null, + configSource: null, + fallbackReason: "agent_runtime_profile_disabled", + adapterConfig: null, + }; + } + + return { + requested, + requestedBy, + applied: requested, + configSource: runtimeProfile.configured ? "agent_runtime" : "adapter_default", + fallbackReason: null, + adapterConfig: { + ...parseObject(adapterProfile.adapterConfig), + ...runtimeProfile.adapterConfig, + }, + }; +} + +export function mergeModelProfileAdapterConfig(input: { + baseConfig: Record; + modelProfile: ModelProfileApplication; + issueAdapterConfig: Record | null | undefined; +}): Record { + return { + ...input.baseConfig, + ...(input.modelProfile.adapterConfig ?? {}), + ...(input.issueAdapterConfig ?? {}), + }; +} + +function modelProfileRunMetadata( + modelProfile: ModelProfileApplication, +): Record | null { + if (!modelProfile.requested) return null; + return { + requested: modelProfile.requested, + requestedBy: modelProfile.requestedBy, + applied: modelProfile.applied, + configSource: modelProfile.configSource, + fallbackReason: modelProfile.fallbackReason, + }; +} + +function mergeModelProfileRunMetadata( + resultJson: Record | null, + modelProfile: ModelProfileApplication, +): Record | null { + const metadata = modelProfileRunMetadata(modelProfile); + if (!metadata) return resultJson; + return { + ...(resultJson ?? {}), + modelProfile: metadata, + }; +} + export function summarizeHeartbeatRunContextSnapshot( contextSnapshot: Record | null | undefined, ): Record | null { @@ -933,6 +1092,7 @@ export function summarizeHeartbeatRunContextSnapshot( "wakeReason", "wakeSource", "wakeTriggerDetail", + "modelProfile", ] as const; for (const key of allowedKeys) { @@ -1558,6 +1718,7 @@ function enrichWakeContextSnapshot(input: { if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) { contextSnapshot.wakeTriggerDetail = triggerDetail; } + normalizeModelProfileWakeContext({ contextSnapshot, payload }); return { contextSnapshot, @@ -4971,9 +5132,42 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) workspaceConfig: existingExecutionWorkspace?.config ?? null, mode: effectiveExecutionWorkspaceMode, }); - const mergedConfig = issueAssigneeOverrides?.adapterConfig - ? { ...persistedWorkspaceManagedConfig, ...issueAssigneeOverrides.adapterConfig } - : persistedWorkspaceManagedConfig; + let adapterModelProfiles: AdapterModelProfileDefinition[] = []; + let profileResolutionFallbackReason: string | null = null; + try { + adapterModelProfiles = await listAdapterModelProfiles(agent.adapterType); + } catch (error) { + profileResolutionFallbackReason = "adapter_profile_resolution_failed"; + logger.warn( + { + err: error, + companyId: agent.companyId, + agentId: agent.id, + adapterType: agent.adapterType, + runId: run.id, + }, + "Failed to resolve adapter model profiles; falling back to primary adapter config", + ); + } + const modelProfileApplication = resolveModelProfileApplication({ + adapterModelProfiles, + agentRuntimeConfig: agent.runtimeConfig, + issueModelProfile: issueAssigneeOverrides?.modelProfile ?? null, + contextSnapshot: context, + profileResolutionFallbackReason, + }); + const modelProfileMetadata = modelProfileRunMetadata(modelProfileApplication); + if (modelProfileMetadata) { + context.paperclipModelProfile = modelProfileMetadata; + if (modelProfileApplication.requested) context.modelProfile = modelProfileApplication.requested; + } else { + delete context.paperclipModelProfile; + } + const mergedConfig = mergeModelProfileAdapterConfig({ + baseConfig: persistedWorkspaceManagedConfig, + modelProfile: modelProfileApplication, + issueAdapterConfig: issueAssigneeOverrides?.adapterConfig ?? null, + }); const configSnapshot = buildExecutionWorkspaceConfigSnapshot(mergedConfig, selectedEnvironmentId); const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const { resolvedConfig, secretKeys } = await resolveExecutionRunAdapterConfig({ @@ -5534,12 +5728,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (key in meta.env) meta.env[key] = "***REDACTED***"; } } + const modelProfileMetadata = modelProfileRunMetadata(modelProfileApplication); await appendRunEvent(currentRun, seq++, { eventType: "adapter.invoke", stream: "system", level: "info", message: "adapter invocation", - payload: meta as unknown as Record, + payload: { + ...(meta as unknown as Record), + ...(modelProfileMetadata ? { modelProfile: modelProfileMetadata } : {}), + }, }); }; @@ -5722,11 +5920,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const persistedResultJson = mergeHeartbeatRunResultJson( mergeRunStopMetadataForAgent(agent, outcome, { - resultJson: mergeAdapterRecoveryMetadata({ - resultJson: adapterResult.resultJson ?? null, - errorFamily: adapterResult.errorFamily ?? null, - retryNotBefore: adapterResult.retryNotBefore ?? null, - }), + resultJson: mergeModelProfileRunMetadata( + mergeAdapterRecoveryMetadata({ + resultJson: adapterResult.resultJson ?? null, + errorFamily: adapterResult.errorFamily ?? null, + retryNotBefore: adapterResult.retryNotBefore ?? null, + }), + modelProfileApplication, + ), errorCode: runErrorCode, errorMessage: runErrorMessage, }),