import { createHmac } from "node:crypto"; import { and, eq, inArray } from "drizzle-orm"; import { agents, companies, companySecrets, type Db } from "@paperclipai/db"; import type { AdapterEnvironmentTestResult } from "@paperclipai/adapter-utils"; import { ADAPTER_AUTH_MISSING_CHECK_CODE, aiRuntimeConnectionBindingSchema } from "@paperclipai/shared"; import { isForbiddenConfigEnvKey, parseObject } from "@paperclipai/adapter-utils/server-utils"; import { requireServerAdapter } from "../adapters/index.js"; import { agentEnvironmentTestService } from "./agent-environment-test.js"; import { aiConnectionRouterService } from "./ai-connection-router.js"; import { aiConnectionService } from "./ai-connections.js"; import { withManagedAiProbe, stripAiAuthBindings } from "./ai-connection-runtime.js"; import { PLUGIN_RUNNER_AI_CONNECTION_REQUIRED, pluginEnvironmentRunnerTaskDriver, pluginRunnerAiConnectionRequiredFailure } from "./plugin-runner-ai-connection.js"; import { environmentService } from "./environments.js"; import { secretService } from "./secrets.js"; import { instanceSettingsService } from "./instance-settings.js"; import { assertProviderApiPlacementPolicy, resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; import { museRunnerBroker } from "./muse-runner-broker.js"; import { RUNNER_REVISION_MISMATCH_CODE } from "./runner-revision-mismatch.js"; import { dotRunnerBroker } from "./dot-runner-broker.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; import type { LifecycleAgent } from "../modules/agent-lifecycle/index.js"; import { resolveDecisionSigningSecret } from "./decision-signing.js"; import { sandboxBinding } from "./environment-probe.js"; import { serverVersion } from "../version.js"; /** * Digest of what a passed harness test proves: the agent's fully resolved * effective configuration (every value the test used, including cwd and * resolved credentials), the AI account and credential generation it selected, * its environment, and the runner image it ran in. Keyed with the instance's * signing secret, so the stored value never verifies a guessed credential * offline. Null when the runner image has no immutable identity: such a pass * is never reused. */ export function harnessVerificationDigest(input: { adapterType: string; config: Record; account: unknown; environmentId: string | null; runner: Record | null; }): string | null { if (!input.runner) return null; const canonical = JSON.stringify({ v: 2, adapterType: input.adapterType, config: input.config, account: input.account ?? null, environmentId: input.environmentId, runner: input.runner }, (_key, value: unknown) => value && typeof value === "object" && !Array.isArray(value) && !(value instanceof Date) ? Object.fromEntries(Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b))) : value); return `v2:hmac-sha256:${createHmac("sha256", resolveDecisionSigningSecret()).update("harness-verification-v2:").update(canonical).digest("hex")}`; } export function assertHarnessTestPassed(result: AdapterEnvironmentTestResult) { const verified = result.checks.some(check => check.level === "info" && (/hello_probe_(passed|succeeded)$/.test(check.code) || check.code === "ai_connection_api_key_reverified" || check.code === "copilot_metadata_verified" || (result.adapterType === "paperclip_runner" && check.code === "cursor_cloud_account_ready"))); const authenticationFailed = result.checks.some(check => check.code === ADAPTER_AUTH_MISSING_CHECK_CODE || check.code === "COPILOT_AUTH_REQUIRED" || /_hello_probe_auth_required$/.test(check.code)); const incomplete = !verified && (result.status === "warn" || result.checks.some(check => check.code.includes("hello_probe"))); if (result.status === "fail" || authenticationFailed || incomplete) { const has = (suffix: string) => result.checks.some(check => check.code.endsWith(suffix)); const code = result.checks.some(check => check.code === PLUGIN_RUNNER_AI_CONNECTION_REQUIRED) ? PLUGIN_RUNNER_AI_CONNECTION_REQUIRED : result.checks.some(check => check.code === RUNNER_REVISION_MISMATCH_CODE) ? RUNNER_REVISION_MISMATCH_CODE : result.checks.some(check => check.level === "error" && check.retryable === true) ? "test_environment_unavailable" : authenticationFailed ? "harness_auth_failed" : has("_hello_probe_model_unavailable") || has("_model_invalid") ? "harness_model_unavailable" : has("_command_unresolvable") ? "harness_command_unavailable" : has("_cwd_invalid") ? "harness_cwd_invalid" : has("_hello_probe_timed_out") ? "harness_probe_timed_out" : has("_hello_probe_unexpected_output") ? "harness_probe_unexpected_output" : "harness_test_failed"; throw Object.assign(new Error("The harness test failed"), { code }); } } export function agentHarnessVerificationService(db: Db, manager: PluginWorkerManager) { const tests = agentEnvironmentTestService(db, manager); const secrets = secretService(db); const environments = environmentService(db); async function verify(snapshot: LifecycleAgent): Promise<"complete" | "pending" | "waiting"> { const [agent] = await db.select().from(agents).where(and(eq(agents.id, snapshot.id), eq(agents.lifecycleVersion, snapshot.lifecycleVersion))); if (!agent) return "pending"; if (agent.adapterType === "paperclip_runner" && ["openai_dot", "grok_bot"].includes(String(agent.adapterConfig.provider))) { const binding = await dotRunnerBroker(db).bindingForAgent(agent.companyId, agent.id); return binding?.status === "ready" && binding.wakeVerified && binding.id === agent.adapterConfig[agent.adapterConfig.provider === "grok_bot" ? "externalBindingId" : "dotBindingId"] ? "complete" : "pending"; } if (agent.adapterType === "paperclip_runner" && agent.adapterConfig.provider === "muse") { const binding = await museRunnerBroker(db).bindingForAgent(agent.companyId,agent.id); // A successful read of these external facts has no retry deadline. The // real confirmation/configuration producer rechecks this phase durably. return binding?.status === "ready" && binding.paired && binding.receiverDetected && binding.backgroundReplyVerified && binding.id === agent.adapterConfig.museBindingId ? "complete" : "waiting"; } const providerApi = resolveProviderApiPlacement({ agent }); if (providerApi) assertProviderApiPlacementPolicy(await instanceSettingsService(db).getGeneral()); const [company] = await db.select().from(companies).where(eq(companies.id, agent.companyId)); const responsibleUserId = agent.lifecycleOperation?.responsibleUserId ?? company.defaultResponsibleUserId; const context = { consumerType: "agent" as const, consumerId: agent.id, actorType: "system" as const, actorId: "agent-lifecycle", responsibleUserId }; let binding = agent.runtimeConfig.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined; let config = agent.adapterConfig; if (binding?.mode === "router") { const selected = await aiConnectionRouterService(db, manager).resolve({ companyId: agent.companyId, poolId: binding.connectionId, agentId: agent.id, userId: responsibleUserId, adapterType: agent.adapterType, taskKey: `lifecycle:${agent.lifecycleOperation!.id}` }); binding = selected.binding; config = { ...config, ...selected.runtimeConfig }; } if (binding) config = { ...config, env: stripAiAuthBindings(config.env) }; config = (await secrets.resolveAdapterConfigForRuntime(agent.companyId, config, context, { adapterType: agent.adapterType })).config; let runner: Record | null = providerApi ? { kind: "provider_api" } : { kind: "local", version: serverVersion }; const environmentId = providerApi ? null : await tests.resolveAdapterTestEnvironmentId(agent.companyId, agent.defaultEnvironmentId, agent.adapterType === "paperclip_runner" && config.provider === "acpx" && config.acpxAgent === "copilot"); if (environmentId) { await tests.assertAdapterTestEnvironmentForCompany(agent.companyId, environmentId, agent.adapterType); const environment = await environments.getById(environmentId); if (environment && environment.driver !== "local") { const image = environment.driver === "sandbox" ? await sandboxBinding(db, environment, agent.companyId).catch(() => null) : null; // Only an immutable image digest names the image a test ran in; a snapshot // name may be re-pointed without the allocation pinning it, so it never counts. const immutable = image && "image" in image; runner = immutable ? { kind: "sandbox", environmentId, ...image } : null; } else if (environment && (await instanceSettingsService(db).getExperimental()).enableManagedSandboxOnly === true) { // The test runs in the managed sandbox instead, whose image is not named here. runner = null; } // Run admission refuses these agents before any lease, so verification must too. const preconfiguredHome = agent.adapterType === "paperclip_runner" && config.homeMode === "preconfigured"; if (agent.adapterType === "paperclip_runner" && environment && (preconfiguredHome || !binding) && await pluginEnvironmentRunnerTaskDriver(db, environment)) throw pluginRunnerAiConnectionRequiredFailure(preconfiguredHome); const env = Object.fromEntries(Object.entries(binding ? stripAiAuthBindings(environment?.envVars) : parseObject(environment?.envVars)) .filter(([key]) => !isForbiddenConfigEnvKey(key))); const resolved = await secrets.resolveEnvBindings(agent.companyId, env, { ...context, consumerType: "environment", consumerId: environmentId }); config = { ...config, env: { ...resolved.env, ...parseObject(config.env) } }; } const selectAccount = () => aiConnectionService(db).select({ companyId: agent.companyId, userId: responsibleUserId, agentId: agent.id, adapterType: agent.adapterType, model: config.model, runnerProvider: config.provider, acpxAgent: config.acpxAgent, binding: binding!, }); const selected = binding ? await selectAccount() : null; // The account and credential generation the binding selected, never the // credential: the selected connection and grant, and the current version // (and AI session epoch) of every secret its credential reads. Timestamps // that change on every use are left out, so a pass stays reusable. const connection = selected?.connection; const credentialRefs = [...(connection?.credentialSecretRefs ?? []), ...(selected?.grant?.credentialSecretRefs ?? [])]; const credentialSecretIds = [...new Set(credentialRefs.map(ref => ref.secretId))]; const readCredentialVersions = async () => credentialSecretIds.length ? await db.select({ id: companySecrets.id, latestVersion: companySecrets.latestVersion, aiSessionEpoch: companySecrets.aiSessionEpoch }) .from(companySecrets).where(and(eq(companySecrets.companyId, agent.companyId), inArray(companySecrets.id, credentialSecretIds))) .then(rows => rows.sort((a, b) => a.id.localeCompare(b.id))) : []; const accountWith = (credentialVersions: Awaited>) => selected ? { ...selected.attribution, credentialSecretRefs: credentialRefs, externalCredential: connection?.externalCredential ?? null, credentialVersions } : null; const credentialVersions = await readCredentialVersions(); // A configuration that already passed in this runner image is verified: // a restart or a return to verifying never boots another test sandbox. // A local probe without a managed AI connection can use the host's own // credentials (environment variables, a CLI login) that no saved // configuration names, so such a pass is never reused. if (runner?.kind === "local" && !binding) runner = null; const digest = harnessVerificationDigest({ adapterType: agent.adapterType, config, account: accountWith(credentialVersions), environmentId, runner }); if (digest && agent.harnessVerifiedDigest === digest) return "complete"; const target = providerApi ? { executionTarget: null, environmentName: null, fallbackChecks: [], release: async (_status?: "released" | "failed") => {} } : await tests.resolveAdapterTestExecutionContext({ agentId: agent.id, companyId: agent.companyId, adapterType: agent.adapterType, environmentId }); let status: "released" | "failed" = "failed"; // A holder, so the account the probe callback records survives control-flow narrowing. const probed: { account: { connectionId: string; grantId: string } | null } = { account: null }; try { if (target.fallbackChecks.length) throw Object.assign(new Error("The test environment is unavailable"), { code: "test_environment_unavailable" }); const input = { companyId: agent.companyId, adapterType: agent.adapterType, config, executionTarget: target.executionTarget, environmentName: target.environmentName }; const result = binding ? await withManagedAiProbe(db, { ...input, agentId: agent.id, responsibleUserId, binding }, managed => { // The account this probe actually uses; the recorded digest must describe it. probed.account = { connectionId: managed.attribution.connectionId, grantId: managed.attribution.grantId }; return tests.testManagedEnvironment(agent.adapterType, { ...input, config: managed.config }, binding!, managed, agent.id); }) : await requireServerAdapter(agent.adapterType).testEnvironment(input); assertHarnessTestPassed(result); status = "released"; // A passing subscription probe can save the credential it refreshed, which // advances the version but keeps the AI session epoch; a person's rotation // bumps the epoch. Record the generation the probe left, unless any epoch // moved: then the pass is not recorded and the next check tests again. const verifiedVersions = await readCredentialVersions(); const sameEpochs = verifiedVersions.length === credentialVersions.length && verifiedVersions.every((row, index) => row.id === credentialVersions[index]!.id && row.aiSessionEpoch === credentialVersions[index]!.aiSessionEpoch); // The managed probe selects the account again when it starts. Record the // pass only for the account it actually tested. const sameAccount = !selected || (probed.account?.connectionId === selected.attribution.connectionId && probed.account?.grantId === selected.attribution.grantId); const verifiedDigest = digest && sameEpochs && sameAccount ? harnessVerificationDigest({ adapterType: agent.adapterType, config, account: accountWith(verifiedVersions), environmentId, runner }) : null; // Record the pass before the lifecycle step completes, so a restart in // between reuses it instead of testing again. if (verifiedDigest) await db.update(agents).set({ harnessVerifiedDigest: verifiedDigest, harnessVerifiedAt: new Date() }) .where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, snapshot.lifecycleVersion))); return "complete"; } finally { await target.release(status); } } return { verify }; }