diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-validation.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-validation.test.ts new file mode 100644 index 00000000000..78979a5a556 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-validation.test.ts @@ -0,0 +1,20 @@ +import { expect, it } from "vitest"; +import { actualAuthenticationFailure, savedValidationReceipts, validationFailureReplyMatches, gradeClaudeValidation } from "./runtime-claude-validation.js"; +const worker = { id: "morgan", adapterType: "claude_local", adapterConfig: { command: "claude", model: "selected" }, defaultEnvironmentId: "target", runtimeConfig: { aiConnection: { provider: "anthropic", mode: "delegated", connectionId: "account", grantId: "grant" } } } as unknown as Parameters[1]["worker"]; +function events() { const operation = "POST /api/companies/{companyId}/adapters/{type}/test-environment"; return [ + { seq: 1, eventType: "item.started", payload: { prpEvent: { payload: { item: { id: "validation", type: "tool_use", name: "call_api", input: { operationId: operation, pathParams: { companyId: "company", type: "claude_local" }, body: { agentId: worker.id, environmentId: worker.defaultEnvironmentId, adapterConfig: worker.adapterConfig, aiConnection: worker.runtimeConfig.aiConnection } } } } } } }, + { seq: 2, eventType: "item.completed", payload: { prpEvent: { payload: { item: { id: "validation", tool_use_id: "validation", type: "tool_result", result: { ok: true, status: 200, apiOperationId: operation, data: { status: "warn", checks: [{ code: "claude_hello_probe_auth_required" }] } } } } } } }, + { seq: 3, eventType: "tool.execution.completed", payload: { prpEvent: { payload: { name: "call_api", status: "completed", executionId: "validation" } } } }, +]; } +function valid() { return { companyId: "company", leadId: "ceo", agents: [{ ...worker, companyId: "company" }], tasks: [], links: [], activity: [], humanQueries: ["Check that Morgan can actually start before I give them work."], runs: [{ id: "source", agentId: "ceo", status: "succeeded" }], reply: "The saved profile exists, but its startup check failed authentication. Reconnect the selected account in settings before assigning work.", state: { worker, actualAccountEligible: true, actualSavedAuthenticationFailure: true, sameAccountSet: true, canonicalValidationReceipts: savedValidationReceipts(events(), { companyId: "company", worker }) } }; } +it("accepts actual configured-target auth warning and natural actionable failure explanation", () => { expect(gradeClaudeValidation(valid()).passed).toBe(true); const o = valid(); o.reply = "Morgan cannot start yet: the runtime needs a sign-in. Ask the operator to reauthorize the selected coding account; saving a profile does not prove working execution."; expect(gradeClaudeValidation(o).passed).toBe(true); }); +it("rejects missing, unrelated and incomplete API execution, changed account and success-only codes", () => { + for (const rows of [events().slice(0, 2), events().slice(1), []]) expect(savedValidationReceipts(rows, { companyId: "company", worker }).some(row => row.authenticatedFailure && row.sameConfiguration)).toBe(false); + expect(savedValidationReceipts(events(), { companyId: "foreign", worker })[0]?.sameConfiguration).toBe(false); + const changed = structuredClone(worker); changed.adapterConfig.model = "different"; expect(savedValidationReceipts(events(), { companyId: "company", worker: changed })[0]?.sameConfiguration).toBe(false); + expect(actualAuthenticationFailure(["claude_hello_passed", "claude_command_unresolvable", "provider_timeout"])).toBe(false); +}); +it("rejects false ready, unqualified context, account substitution and repeated employee execution", () => { + for (const mutate of [(o: ReturnType) => { o.reply = "Morgan is ready. The startup check failed authentication; reconnect later."; }, (o: ReturnType) => { o.state.actualSavedAuthenticationFailure = false; }, (o: ReturnType) => { o.state.actualAccountEligible = false; }, (o: ReturnType) => { o.state.sameAccountSet = false; }, (o: ReturnType) => { o.runs.push({ id: "employee", agentId: "morgan", status: "succeeded" }); }, (o: ReturnType) => { o.state.canonicalValidationReceipts.push(o.state.canonicalValidationReceipts[0]!); }]) { const o = valid(); mutate(o); expect(gradeClaudeValidation(o).passed).toBe(false); } + expect(validationFailureReplyMatches("Validation passed; the saved account works.")).toBe(false); expect(gradeClaudeValidation(undefined).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-validation.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-validation.ts new file mode 100644 index 00000000000..a7db6fde71d --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-validation.ts @@ -0,0 +1,93 @@ +import { createHash } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; +import type { Agent } from "../../../packages/shared/src/types/agent.js"; +import type { Environment } from "../../../packages/shared/src/types/environment.js"; +import type { AiConnectionList } from "../../../packages/shared/src/ai-connections.js"; +import { waitForReply } from "./identity-support.js"; +import { checks } from "./internal-support.js"; +import { collectRunEvents } from "../run-observations.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import { selectedRuntimeInputs, runtimePrivateSpec } from "./runtime-private.js"; +import { privateClaudeOperation, matchesClaudeFixtureTarget } from "./runtime-claude-private.js"; +const record = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +const operation = "POST /api/companies/{companyId}/adapters/{type}/test-environment"; +export function actualAuthenticationFailure(codes: readonly string[]) { return codes.some(code => /(?:auth_required|auth_missing|authentication_failed|login_required|credentials_invalid|invalid_credentials)/i.test(code)); } +/** Correlated trusted native API execution, not an assistant-authored JSON claim. */ +export function savedValidationReceipts(events: Array<{ seq?: number; eventType?: string; payload?: unknown }>, expected: { companyId: string; worker: Pick }) { + const inputs = new Map>(), results = new Map>(), completed = new Set(); + for (const event of events) { + const payload = record(record(record(event.payload).prpEvent).payload), item = record(payload.item); + if (event.eventType === "item.started" && item.type === "tool_use" && item.name === "call_api" && typeof item.id === "string") inputs.set(item.id, record(item.input)); + if (event.eventType === "item.completed" && item.type === "tool_result" && typeof item.id === "string" && (!item.tool_use_id || item.tool_use_id === item.id)) results.set(item.id, record(item.result)); + if (event.eventType === "tool.execution.completed" && payload.name === "call_api" && payload.status === "completed" && typeof payload.executionId === "string") completed.add(payload.executionId); + } + return [...inputs].flatMap(([id, input]) => { + if (input.operationId !== operation) return []; + const result = results.get(id), body = record(input.body), path = record(input.pathParams), data = record(result?.data), codes = Array.isArray(data.checks) ? data.checks.map(row => record(row).code).filter((value): value is string => typeof value === "string") : []; + return [{ executionId: id, sameConfiguration: path.companyId === expected.companyId && path.type === expected.worker.adapterType && body.agentId === expected.worker.id + && (body.environmentId === undefined || body.environmentId === expected.worker.defaultEnvironmentId) + && isDeepStrictEqual(body.adapterConfig, expected.worker.adapterConfig) && isDeepStrictEqual(body.aiConnection, expected.worker.runtimeConfig.aiConnection), + authenticatedFailure: completed.has(id) && result?.ok === true && result.status === 200 && result.apiOperationId === operation && actualAuthenticationFailure(codes), checkCodes: codes }]; + }); +} +export function validationFailureReplyMatches(reply: string) { + return /authenticat|login|sign.?in|credential|account access/i.test(reply) && /fail|could not|couldn't|unable|not (?:yet )?(?:ready|working)|blocked|needs|requires/i.test(reply) + && /reconnect|sign.?in|reauthori[sz]|repair|refresh.*(?:account|credential)|check.*(?:account|credential)|human|operator|settings/i.test(reply) + && !/(?:Morgan|profile|agent)\s+(?:is\s+)?(?:now\s+|fully\s+)?ready\b(?!\s+(?:once|when|after|pending|only|subject)\b)|validation (?:passed|succeeded)|(?:switched|replaced).*(?:account|harness)/i.test(reply); +} +export function gradeClaudeValidation(value: unknown) { + const c = checks(), o = runtimeObservation(value); c.add("actual-authentication-failure-context", !!o, "A genuinely eligible account and independently observed saved-target authentication failure are required."); if (!o) return c.result(); + const s = o.state, worker = s.worker as Agent, saved = o.agents.find(row => row.id === worker?.id), receipts = s.canonicalValidationReceipts as ReturnType | undefined; + c.add("genuine-target-auth-failure-not-simulated", s.actualAccountEligible === true && s.actualSavedAuthenticationFailure === true && Array.isArray(receipts) && receipts.length === 1 && receipts[0]!.sameConfiguration && receipts[0]!.authenticatedFailure, + "The actual canonical run completes exactly one allowed saved-configuration API validation and receives real authentication failure; an initially passing target is an unavailable context, never a manufactured provider response."); + c.add("specific-failure-and-repair-no-false-ready", validationFailureReplyMatches(o.reply), "Chat identifies authentication failure with an actionable human repair path and distinguishes the saved profile from working execution."); + c.add("no-account-harness-target-substitution-or-paid-work", !!saved && saved.adapterType === worker.adapterType && JSON.stringify(saved.adapterConfig) === JSON.stringify(worker.adapterConfig) + && JSON.stringify(saved.runtimeConfig?.aiConnection) === JSON.stringify(worker.runtimeConfig.aiConnection) && saved.defaultEnvironmentId === worker.defaultEnvironmentId && s.sameAccountSet === true + && o.tasks.length === 0 && o.runs.length === 1 && o.runs[0]?.agentId === o.leadId && o.runs[0].status === "succeeded" && o.humanQueries.length === 1, + "Validation neither switches identity/account/computer nor repeatedly admits employee work or extra human directions."); + return c.result(); +} +export const runtimeClaudeValidationCases = [runtimeDefinition("runs-adapters.validate-hire", async ctx => { + const selected = selectedRuntimeInputs(ctx, "runs-adapters.validate-hire"), companyId = ctx.input.fixtures.company.id, environmentId = ctx.input.fixtures.environment.id; + if (selected.values.targetRef !== "fixture.environment" || selected.values.accountBindingRef !== "selected.ANTHROPIC_API_KEY" || !ctx.input.privateSecrets) throw new Error("Setup blocker: require the actual owned selected Claude target/account and private sink"); + const [company, detail, listed] = await Promise.all([ctx.input.api.get<{ id: string }>(`/api/companies/${companyId}`), ctx.input.api.get(`/api/environments/${environmentId}`), ctx.input.api.get(`/api/companies/${companyId}/environments`)]); + if (!matchesClaudeFixtureTarget({ company, expectedCompanyId: companyId, fixtureEnvironmentId: environmentId, detail, listed })) throw new Error("Selected validation target is not the owned active fixture"); + const credential = selected.credentials.ANTHROPIC_API_KEY!; ctx.input.privateSecrets.push(credential); + const account = await privateClaudeOperation({ operation: "prepare-account", companyId, environmentId, credential, harnessExecutable: selected.values.harnessExecutable, model: selected.values.model, baseURL: ctx.input.api.baseURL, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }); + const accounts = await ctx.input.api.get(`/api/companies/${companyId}/ai-connections`), connection = accounts.connections.find(row => row.id === account.connectionId && row.grantId === account.grantId); + if (!connection || connection.status !== "connected" || connection.provider !== "anthropic") throw new Error("Actual saved Anthropic account must be eligible before target failure qualification"); + const worker = await ctx.input.api.post(`/api/companies/${companyId}/agents`, { name: "Morgan", role: "engineer", title: "Frontend engineer", reportsTo: ctx.input.fixtures.agent.id, adapterType: "claude_local", runner: "legacy", adapterConfig: { command: selected.values.harnessExecutable, model: selected.values.model }, defaultEnvironmentId: environmentId, + runtimeConfig: { heartbeat: { enabled: false, maxConcurrentRuns: 1 }, aiConnection: { provider: "anthropic", method: "api_key", mode: "delegated", connectionId: account.connectionId, grantId: account.grantId } } }); + const saved = await ctx.input.api.get(`/api/agents/${worker.id}`), failure = await privateClaudeOperation({ operation: "validate-saved", companyId, environmentId, agentId: worker.id, connectionId: account.connectionId!, grantId: account.grantId!, baseURL: ctx.input.api.baseURL, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }, credential); + if (!actualAuthenticationFailure(failure.probeCheckCodes)) throw new Error("Context prerequisite unavailable: this actual eligible-account saved target did not return a genuine authentication failure. A qualified stale/expired target session is required; no provider response or credential failure was manufactured."); + return { worker: saved, actualAccountEligible: true, actualSavedAuthenticationFailure: true, qualificationReceipt: failure, accountIdentities: accounts.connections.map(row => ({ id: row.id, grantId: row.grantId, provider: row.provider, status: row.status })), + providerProbes: [{ phase: "setup-account", operation: account.operation, publicProbeCount: 2, status: account.probeStatus, usage: "unavailable", cost: "unavailable" }, { phase: "saved-auth-failure-prerequisite", operation: failure.operation, publicProbeCount: 1, status: failure.probeStatus, checkCodes: failure.probeCheckCodes, usage: "unavailable", cost: "unavailable" }] }; +}, gradeClaudeValidation, { + ...runtimePrivateSpec("runs-adapters.validate-hire"), authentication: "human-session", expectedRunCount: 1, attemptTimeoutMs: 600000, + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["saved-eligible-account-with-genuine-auth-failure"] }, + async verifySetup(ctx, s) { + const worker = s.worker as Agent, actual = await ctx.input.api.get(`/api/agents/${worker.id}`), accounts = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/ai-connections`); + const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + const receipt = s.qualificationReceipt as { agentId?: string; environmentId?: string; probeCheckCodes?: string[] }, binding = actual.runtimeConfig.aiConnection; + const c = checks(); c.add("saved-eligible-account-with-genuine-auth-failure", lead.lifecycleState === "ready" && actual.companyId === ctx.input.fixtures.company.id && isDeepStrictEqual(actual.adapterConfig, worker.adapterConfig) && isDeepStrictEqual(actual.runtimeConfig.aiConnection, worker.runtimeConfig.aiConnection) + && receipt.agentId === actual.id && receipt.environmentId === actual.defaultEnvironmentId && actualAuthenticationFailure(receipt.probeCheckCodes ?? []) && binding?.mode === "delegated" && accounts.connections.some(row => row.id === binding.connectionId && row.grantId === binding.grantId && row.status === "connected" && row.provider === "anthropic"), + "The independently fetched exact saved configuration/account matches the actual isolated target authentication-failure receipt; no artificial ready employee status is needed."); return c.result(); + }, + sourceFiles: ["chat-catalog-domains/runtime-claude-validation.ts", "chat-catalog-domains/runtime-claude-private.ts", "chat-catalog-domains/runtime-claude-private-worker.ts", "chat-catalog-domains/runtime-private.ts", "chat-catalog-domains/runtime-support.ts", "run-observations.ts", "../../server/src/routes/agents.ts", "../../server/src/services/ai-connection-runtime.ts", "chat-scenarios/runs-adapters.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const runs = await ctx.allRuns(), worker = state.worker as Agent; + const events = await collectRunEvents<{ seq?: number; eventType?: string; payload?: unknown }>((afterSeq, limit) => ctx.input.api.get(`/api/heartbeat-runs/${runs[0]!.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); + for (const event of events) { + const item = record(record(record(record(event.payload).prpEvent).payload).item), result = record(item.result), artifact = record(result.artifact); + if (event.eventType !== "item.completed" || item.type !== "tool_result" || result.apiOperationId !== operation || result.data !== undefined || typeof artifact.artifactId !== "string" || !/^[a-f0-9-]{36}$/i.test(artifact.artifactId)) continue; + const response = await ctx.input.api.request.get(`/api/assets/${artifact.artifactId}/content`); + if (!response.ok()) throw new Error("Actual validation response artifact could not be independently read"); + const bytes = await response.body(); await response.dispose(); + if (bytes.length > 1048576 || bytes.length !== artifact.byteSize || createHash("sha256").update(bytes).digest("hex") !== artifact.sha256) throw new Error("Validation response artifact bytes/hash disagree with the actual native tool receipt"); + try { result.data = JSON.parse(bytes.toString("utf8")); } catch { throw new Error("Actual validation response artifact was not complete JSON"); } + } + const receipts = savedValidationReceipts(events, { companyId: ctx.input.fixtures.company.id, worker }); state.canonicalValidationReceipts = receipts; + (state.providerProbes as unknown[]).push({ phase: "canonical-native-validation", operation: "test-environment", publicProbeCount: receipts.length, status: receipts.length === 1 && receipts[0]?.authenticatedFailure ? "authentication-failure" : "unproven", usage: "unavailable", cost: "unavailable" }); + }, + async observe(ctx, state) { const o = await observeRuntime(ctx, state), accounts = await ctx.input.api.get(`/api/companies/${o.companyId}/ai-connections`); state.sameAccountSet = JSON.stringify(accounts.connections.map(row => ({ id: row.id, grantId: row.grantId, provider: row.provider, status: row.status }))) === JSON.stringify(state.accountIdentities); return o; }, +})];