From b81b95b36ebce082a285509ee1bbd35a8c602b34 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 00:20:51 -0500 Subject: [PATCH] test(evals): author missing Claude account and real hire resumption --- .../runtime-claude-account.test.ts | 13 +++ .../runtime-claude-account.ts | 91 +++++++++++++++++++ ...runtime-claude-private-canary-receipt.json | 6 +- .../runtime-claude-private-worker.ts | 11 ++- .../runtime-claude-private.ts | 6 +- 5 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-claude-account.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.test.ts new file mode 100644 index 00000000000..9539c9f1551 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.test.ts @@ -0,0 +1,13 @@ +import { expect, it } from "vitest"; +import { gradeClaudeAccount, supportedClaudeSetupLink } from "./runtime-claude-account.js"; +function valid() { return { companyId: "company", leadId: "ceo", agents: [{ id: "morgan", name: "Morgan", companyId: "company", role: "engineer", title: "Frontend engineer", lifecycleState: "ready", status: "idle", adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "claude" } }], tasks: [], links: [], humanQueries: ["Hire Morgan as a frontend engineer using Claude Code.", "The Anthropic account is connected now. Continue the same Morgan frontend engineer hire using Claude Code."], reply: "", activity: [{ id: "hire", entityId: "morgan", action: "agent.hire_created", actorType: "agent", actorId: "ceo", runId: "second" }], runs: [{ id: "first", agentId: "ceo", status: "succeeded" }, { id: "second", agentId: "ceo", status: "succeeded" }], state: { initialAnthropicAccounts: 0, actualInstalledClaude: true, supportedSetupVisible: true, waitingReply: "Morgan's frontend hire is waiting for an Anthropic account. Connect the account using this setup request before Claude can start.", noUsableMorganBeforeHuman: true, accountOwnerIsHuman: true, humanAccountConnected: true, continuationMessageBound: true, beforeMorganId: "morgan", actualAccountBinding: true, savedValidation: { probeStatus: "pass", agentId: "morgan" }, continuationReply: "Morgan is hired and the selected account is connected.", initialRunId: "first", continuationRunId: "second" } }; } +it("accepts preserved waiting hire followed by the actual human account and attributed continuation", () => { expect(gradeClaudeAccount(valid()).passed).toBe(true); const o = valid(); o.state.waitingReply = "I cannot finish the requested frontend hire until your Claude coding account is connected. Please use the supported setup flow."; o.state.continuationReply = "The frontend engineer has been added with the connected account."; expect(gradeClaudeAccount(o).passed).toBe(true); }); +it("accepts a truthful future qualification instead of treating once-connected as current readiness", () => { const o = valid(); o.state.waitingReply = "Morgan is ready once the missing Anthropic account setup is complete; until then this hire is waiting for your connection."; expect(gradeClaudeAccount(o).passed).toBe(true); }); +it("rejects ready-before-access, wrong human, lost identity, wrong phase and substituted harness", () => { + for (const mutate of [(o: ReturnType) => { o.state.waitingReply = "Morgan is ready for work; connect the missing Anthropic account later."; }, (o: ReturnType) => { o.state.noUsableMorganBeforeHuman = false; }, (o: ReturnType) => { o.state.accountOwnerIsHuman = false; }, (o: ReturnType) => { o.state.beforeMorganId = "abandoned-original"; }, (o: ReturnType) => { o.state.continuationMessageBound = false; }, (o: ReturnType) => { o.agents[0]!.adapterConfig.acpxAgent = "codex"; }, (o: ReturnType) => { o.state.actualAccountBinding = false; }]) { const o = valid(); mutate(o); expect(gradeClaudeAccount(o).passed).toBe(false); } + expect(gradeClaudeAccount(undefined).passed).toBe(false); +}); +it("accepts same-resource account setup links and rejects foreign company/provider routes", () => { + for (const href of ["/ACORN/apps/connect?source=anthropic", "http://fixture.invalid/ACORN/apps/connect?provider=claude", "/ACORN/company/settings/connections"]) expect(supportedClaudeSetupLink(href, "http://fixture.invalid", "ACORN")).toBe(true); + for (const href of ["https://foreign.invalid/ACORN/apps/connect?source=anthropic", "/BIRCH/apps/connect?source=anthropic", "/ACORN/apps/connect?source=openai"]) expect(supportedClaudeSetupLink(href, "http://fixture.invalid", "ACORN")).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts new file mode 100644 index 00000000000..527d851f7ea --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts @@ -0,0 +1,91 @@ +import { expect } from "@playwright/test"; +import type { Environment } from "../../../packages/shared/src/types/environment.js"; +import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js"; +import type { Agent } from "../../../packages/shared/src/types/agent.js"; +import type { ActivityEvent } from "../../../packages/shared/src/types/activity.js"; +import type { AiConnectionList } from "../../../packages/shared/src/ai-connections.js"; +import { agentHarnessType } from "../../../packages/shared/src/agent-runner.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, type Entity } from "./internal-support.js"; +import { waitForReply } from "./identity-support.js"; +import { matchesClaudeFixtureTarget, privateClaudeOperation, type ClaudePrivateReceipt } from "./runtime-claude-private.js"; +import { selectedRuntimeInputs, runtimePrivateSpec } from "./runtime-private.js"; +import { openRuntimeAgentProfile, observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js"; +const continueHire = "The Anthropic account is connected now. Continue the same Morgan frontend engineer hire using Claude Code."; +export function supportedClaudeSetupLink(href: string, origin: string, prefix: string) { + try { const url = new URL(href, origin); return url.origin === origin && url.pathname.startsWith(`/${prefix}/`) && (url.pathname.endsWith("/company/settings/connections") || url.pathname.endsWith("/apps/connect") && ["anthropic", "claude"].includes(url.searchParams.get("source") ?? url.searchParams.get("provider") ?? "")); } catch { return false; } +} +export function gradeClaudeAccount(value: unknown) { + const c = checks(), o = runtimeObservation(value); + c.add("genuine-missing-account-and-human-resumption-evidence", !!o, "Installed Claude, an actually absent eligible account, exact canonical waiting reply, real human connection and separately attributed continuation are required."); if (!o) return c.result(); + const s = o.state, workers = o.agents.filter(row => row.name === "Morgan"), worker = workers[0] as (RuntimeAgent & Pick) | undefined, first = String(s.waitingReply), validation = s.savedValidation as ClaudePrivateReceipt | undefined; + c.add("truthful-account-setup-with-original-hire-details", s.initialAnthropicAccounts === 0 && s.actualInstalledClaude === true && s.supportedSetupVisible === true + && /Morgan|frontend|this hire|requested (?:hire|engineer)/i.test(first) && /Claude|Anthropic|coding|provider|model/i.test(first) && /account|connect|sign.?in|authenticat/i.test(first) && /need|wait|setup|set up|missing|before|blocked|to (?:finish|continue|complete)|requires|until|once|when/i.test(first) + && !/Morgan\s+(?:(?:is|now|already)\s+){1,2}(?:fully\s+)?ready\b(?!\s+(?:once|when|after|pending|only once|only after|subject to)\b)|\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:connected|authorized|authenticated)\b/i.test(first) && s.noUsableMorganBeforeHuman === true, + "The first reply preserves the Morgan/Claude request and directs the responsible human to actual supported setup, without a ready hire or alternate harness before account access exists."); + c.add("actual-human-account-followed-by-one-same-hire", s.accountOwnerIsHuman === true && s.humanAccountConnected === true && s.continuationMessageBound === true && workers.length === 1 && !!worker + && worker.companyId === o.companyId && worker.lifecycleState === "ready" && ["idle", "active"].includes(worker.status ?? "") && worker.role === "engineer" && /frontend|front.end/i.test(`${worker.title ?? ""} ${worker.capabilities ?? ""}`) + && agentHarnessType(worker.adapterType ?? "", worker.adapterConfig) === "claude_local" && (!s.beforeMorganId || s.beforeMorganId === worker.id) + && s.actualAccountBinding === true && validation?.probeStatus === "pass" && validation.agentId === worker.id, + "The signed-in human really connects the selected Anthropic account; the same preserved request then produces one usable permanent frontend Claude hire, preserving any initial pending identity."); + c.add("phase-attributed-durable-reply-and-no-unrequested-work", /Morgan|frontend|engineer|hire/i.test(String(s.continuationReply)) && /creat|hir|saved|added|complete|ready|connected/i.test(String(s.continuationReply)) + && o.humanQueries.length === 2 && o.humanQueries[1] === continueHire && o.tasks.length === 0 && o.runs.length === 2 && o.runs.every(row => row.status === "succeeded" && row.agentId === o.leadId) + && o.activity.some(row => ["agent.created", "agent.hire_created"].includes(row.action ?? "") && row.entityId === worker?.id && row.actorId === o.leadId && row.actorType === "agent" && [s.initialRunId, s.continuationRunId].includes((row as unknown as ActivityEvent).runId)), + "Only the two declared real human turns run; the second turn's actual native reply reports the saved hire and no employee work or duplicate agent is introduced."); + return c.result(); +} +export const runtimeClaudeAccountCases = [runtimeDefinition("agents.hire-needs-account", async ctx => { + const selected = selectedRuntimeInputs(ctx, "agents.hire-needs-account"), companyId = ctx.input.fixtures.company.id; + if (selected.values.targetRef !== "fixture.environment" || selected.values.accountBindingRef !== "selected.ANTHROPIC_API_KEY" || !ctx.input.privateSecrets) throw new Error("Setup blocker: require qualified owned fixture.environment, selected Anthropic credential and private redaction sink"); + const [detail, listed, company, accounts, agents] = await Promise.all([ctx.input.api.get(`/api/environments/${ctx.input.fixtures.environment.id}`), ctx.input.api.get(`/api/companies/${companyId}/environments`), ctx.input.api.get<{ id: string }>(`/api/companies/${companyId}`), ctx.input.api.get(`/api/companies/${companyId}/ai-connections`), ctx.input.api.get(`/api/companies/${companyId}/agents`)]); + if (!matchesClaudeFixtureTarget({ company, expectedCompanyId: companyId, fixtureEnvironmentId: ctx.input.fixtures.environment.id, detail, listed }) || accounts.connections.some(row => row.provider === "anthropic" && row.status === "connected") || agents.some(row => row.name === "Morgan")) throw new Error("The missing-account context requires actual installed local Claude, absent Anthropic access and absent Morgan"); + const key = selected.credentials.ANTHROPIC_API_KEY!; ctx.input.privateSecrets.push(key); + const available = await privateClaudeOperation({ operation: "check-harness", harnessExecutable: selected.values.harnessExecutable, companyId, environmentId: detail.id, baseURL: ctx.input.api.baseURL, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }, key); + await ctx.input.api.patch(`/api/companies/${companyId}`, { requireBoardApprovalForNewAgents: false }); + return { selectedEnvironmentId: detail.id, initialAnthropicAccounts: 0, actualInstalledClaude: available.probeStatus === "pass" }; +}, gradeClaudeAccount, { + ...runtimePrivateSpec("agents.hire-needs-account"), authentication: "human-session", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 900000, + sourceFiles: ["chat-catalog-domains/runtime-claude-account.ts", "chat-catalog-domains/runtime-claude-private.ts", "chat-catalog-domains/runtime-claude-private-worker.ts", "chat-catalog-domains/runtime-private.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const before = await observeRuntime(ctx, state), source = ctx.input.page.url(), firstRun = before.runs[0]!; + state.initialRunId = firstRun.id; + const comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + state.waitingReply = comments.filter(row => row.authorAgentId === before.leadId && row.createdByRunId === firstRun.id).map(row => row.body).join("\n"); + const pending = before.agents.filter(row => row.name === "Morgan"); state.beforeMorganId = pending[0]?.id; + state.noUsableMorganBeforeHuman = pending.length <= 1 && pending.every(row => row.lifecycleState !== "ready" && agentHarnessType(row.adapterType ?? "", row.adapterConfig) === "claude_local"); + const target = before.links.find(href => supportedClaudeSetupLink(href, before.origin, ctx.input.fixtures.company.issuePrefix!)); + const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + if (target) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(target)}]`).first().click(); + else { + const requests = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`); + const request = requests.find(row => row.kind === "connection_intent" && row.status === "pending" && row.sourceRunId === firstRun.id && row.addresseeUserId === session.user.id && row.payload.serviceSlug === "anthropic"); + if (!request) throw new Error("The canonical waiting response exposed neither a supported account setup link nor an actual addressed Anthropic setup request"); + await ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(request.id)}]`).getByRole("button", { name: /Connect|Set up|Continue/i }).first().click(); + state.setupRequestId = request.id; + } + await expect(ctx.input.page.getByRole("heading", { name: /connections|Connect.*Anthropic|Connect a model provider/i }).first()).toBeVisible(); state.supportedSetupVisible = true; + const selected = selectedRuntimeInputs(ctx, "agents.hire-needs-account"); + const account = await privateClaudeOperation({ operation: "prepare-account", companyId: before.companyId, environmentId: String(state.selectedEnvironmentId), credential: selected.credentials.ANTHROPIC_API_KEY!, harnessExecutable: selected.values.harnessExecutable, model: selected.values.model, baseURL: ctx.input.api.baseURL, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }); + state.connectionId = account.connectionId; state.grantId = account.grantId; + const accounts = await ctx.input.api.get(`/api/companies/${before.companyId}/ai-connections`), saved = accounts.connections.find(row => row.id === account.connectionId && row.grantId === account.grantId); + state.accountOwnerIsHuman = saved?.ownerUserId === session.user.id; state.humanAccountConnected = saved?.status === "connected" && saved.provider === "anthropic" && account.probeStatus === "pass"; + if (!state.humanAccountConnected || !state.accountOwnerIsHuman) throw new Error("The actual human account setup did not establish usable selected Anthropic access"); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByText(saved!.name, { exact: true }).first()).toBeVisible(); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, continueHire); + const settled = await pollUntil({ label: "Actual explicit human account confirmation resumes the preserved Claude hire", deadlineAt: Date.now() + 240000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), agents: await ctx.input.api.get(`/api/companies/${before.companyId}/agents`) }; }, + accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.every(row => row.status === "succeeded") && v.agents.filter(row => row.name === "Morgan").length === 1 && v.agents.find(row => row.name === "Morgan")?.lifecycleState === "ready", + reject: v => v.runs.length > 2 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Account resumption failed or admitted undeclared duplicate work" : undefined }); + const run = settled.runs.find(row => row.id !== firstRun.id)!, next = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`); + const after = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = after.filter(row => !row.authorAgentId && row.body === continueHire); + state.continuationMessageBound = human.length === 1 && next.contextSnapshot?.wakeCommentIds?.length === 1 && next.contextSnapshot.wakeCommentIds[0] === human[0]?.id && next.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(continueHire) === true; + state.continuationRunId = run.id; state.continuationReply = after.filter(row => row.authorAgentId === before.leadId && row.createdByRunId === run.id).map(row => row.body).join("\n"); + const worker = settled.agents.find(row => row.name === "Morgan")!; state.worker = worker; + state.savedValidation = await privateClaudeOperation({ operation: "validate-saved", companyId: before.companyId, environmentId: String(state.selectedEnvironmentId), agentId: worker.id, connectionId: String(state.connectionId), grantId: String(state.grantId), baseURL: ctx.input.api.baseURL, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }, selected.credentials.ANTHROPIC_API_KEY); + }, + async observe(ctx, state) { const o = await observeRuntime(ctx, state), worker = o.agents.find(row => row.name === "Morgan"), binding = worker?.runtimeConfig?.aiConnection as { provider?: string; mode?: string; connectionId?: string; grantId?: string } | undefined; + const accounts = await ctx.input.api.get(`/api/companies/${o.companyId}/ai-connections`); + state.actualAccountBinding = binding?.provider === "anthropic" && accounts.connections.some(row => row.id === state.connectionId && row.grantId === state.grantId && row.status === "connected" && (binding.mode === "responsible_user" ? row.isDefault : binding.connectionId === row.id && binding.grantId === row.grantId)); return o; }, + async verifyResult(ctx, state, value) { const o = runtimeObservation(value); if (!o) throw new Error("Missing account-resumption result"); const source = ctx.input.page.url(); await openRuntimeAgentProfile(ctx, state.worker as RuntimeAgent, o); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("heading", { name: "Morgan", exact: true }).first()).toBeVisible(); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-canary-receipt.json b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-canary-receipt.json index 3aa90391e06..f477fa075ea 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-canary-receipt.json +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-canary-receipt.json @@ -18,11 +18,11 @@ "error-context" ], "scannedItems": 13, - "privateValueCount": 24, + "privateValueCount": 30, "findings": [], "sourceSha256": { - "runtime-claude-private.ts": "435ce9b149120cf2c76c7cb309e68dbe1b30403ff2af094be0b96c7e2ac407a1", - "runtime-claude-private-worker.ts": "3ad40e48955e41197334688900a7ec76225d6f9739b88ecd360b0582ba4ac434", + "runtime-claude-private.ts": "e98adfc0895d7b40657426015a52af513f34439d2f76d7cb40e93271808f340d", + "runtime-claude-private-worker.ts": "bfa194e44279c208cdc09fb5c9aaea3bfeb053e91a19925ac80facbd84b7b8c5", "runtime-claude-private-privacy.spec.ts": "19975f6048808e53bdf98633fb7e3b6470bd5336a3141032f80d0237561546c7" }, "configSha256": "890c62895e7b90eb24c2b96a321edc58b2f6b800531c2607a663fa6e4a155554", diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-worker.ts index 749e1e6a036..694a74c6713 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-worker.ts @@ -1,3 +1,6 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import path from "node:path"; import { readFile } from "node:fs/promises"; import { test, type Cookie } from "@playwright/test"; import { matchesClaudeFixtureTarget, type ClaudePrivateRequest, type ClaudePrivateReceipt } from "./runtime-claude-private.js"; @@ -10,7 +13,7 @@ async function main(): Promise { if (attached) throw new Error("Claude credential operation cannot run inside Playwright Test"); const input = JSON.parse(await readFile(process.argv[2]!, "utf8")) as ClaudePrivateRequest & { companyId: string; environmentId: string; baseURL: string; cookies: Cookie[] }; const base = new URL(input.baseURL); - if (base.protocol !== "http:" || base.hostname !== "127.0.0.1" || ![input.companyId, input.environmentId].every(id => /^[a-f0-9-]{36}$/i.test(id)) || !["prepare-account", "validate-saved"].includes(input.operation)) throw new Error("Invalid private Claude operation binding"); + if (base.protocol !== "http:" || base.hostname !== "127.0.0.1" || ![input.companyId, input.environmentId].every(id => /^[a-f0-9-]{36}$/i.test(id)) || !["check-harness", "prepare-account", "validate-saved"].includes(input.operation)) throw new Error("Invalid private Claude operation binding"); const headers = { Cookie: input.cookies.map(row => `${row.name}=${row.value}`).join("; "), Origin: base.origin, "Content-Type": "application/json" }; async function json(pathname: string, body?: unknown): Promise { const target = new URL(pathname, base); if (target.origin !== base.origin || !pathname.startsWith("/api/")) throw new Error("Private Claude request escaped its actual instance"); @@ -19,6 +22,12 @@ async function main(): Promise { } const [environment, inventory, company] = await Promise.all([json(`/api/environments/${input.environmentId}`), json(`/api/companies/${input.companyId}/environments`), json<{ id: string }>(`/api/companies/${input.companyId}`)]); if (!matchesClaudeFixtureTarget({ company, expectedCompanyId: input.companyId, fixtureEnvironmentId: input.environmentId, detail: environment, listed: inventory })) throw new Error("Selected Claude fixture requires the exact owned local target with actual public inventory/company authority"); + if (input.operation === "check-harness") { + if (!path.isAbsolute(input.harnessExecutable)) throw new Error("Claude command qualification requires its actual explicit executable path"); + const version = await promisify(execFile)(input.harnessExecutable, ["--version"], { timeout: 15000, maxBuffer: 16384 }); + if (!/\bClaude Code\b/i.test(version.stdout) || !/\b\d+\.\d+\.\d+\b/.test(version.stdout)) throw new Error("The selected executable did not identify the real installed Claude Code CLI"); + return { processIsolation: true, testRunnerAttached: false, apiStepRecorder: false, operation: input.operation, companyId: input.companyId, environmentId: input.environmentId, probeStatus: "pass", probeCheckCodes: ["actual_claude_code_version_observed"] }; + } let connectionId: string, grantId: string, config: Record, agentId: string | undefined, savedType: string | undefined, savedBinding: AiRuntimeConnectionBinding | undefined; if (input.operation === "prepare-account") { if (!input.credential || !input.harnessExecutable || !input.model) throw new Error("Missing selected Claude credential/command/model"); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private.ts index e8149f64902..748860baa3f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private.ts @@ -15,18 +15,18 @@ export function matchesClaudeFixtureTarget(input: { company: { id: string }; exp } export interface ClaudePrivateReceipt { processIsolation: true; testRunnerAttached: false; apiStepRecorder: false; - operation: "prepare-account" | "validate-saved"; companyId: string; environmentId: string; connectionId: string; grantId: string; agentId?: string; + operation: "check-harness" | "prepare-account" | "validate-saved"; companyId: string; environmentId: string; connectionId?: string; grantId?: string; agentId?: string; probeStatus: "pass" | "warn" | "fail"; probeCheckCodes: string[]; } export function validateClaudePrivateReceipt(value: unknown, expected: Pick): ClaudePrivateReceipt { const r = value as ClaudePrivateReceipt; if (!r || r.processIsolation !== true || r.testRunnerAttached !== false || r.apiStepRecorder !== false || r.operation !== expected.operation || r.companyId !== expected.companyId || r.environmentId !== expected.environmentId || Object.keys(r).some(key => !["processIsolation", "testRunnerAttached", "apiStepRecorder", "operation", "companyId", "environmentId", "connectionId", "grantId", "agentId", "probeStatus", "probeCheckCodes"].includes(key)) - || ![r.companyId, r.environmentId, r.connectionId, r.grantId, ...(r.agentId ? [r.agentId] : [])].every(id => typeof id === "string" && /^[a-f0-9-]{36}$/i.test(id)) + || ![r.companyId, r.environmentId, ...(r.operation === "check-harness" ? [] : [r.connectionId, r.grantId]), ...(r.agentId ? [r.agentId] : [])].every(id => typeof id === "string" && /^[a-f0-9-]{36}$/i.test(id)) || !["pass", "warn", "fail"].includes(r.probeStatus) || !Array.isArray(r.probeCheckCodes) || r.probeCheckCodes.length > 100 || !r.probeCheckCodes.every(code => /^[a-z0-9_.-]{1,100}$/i.test(code))) throw new Error("Invalid closed Claude qualification receipt; private diagnostics withheld"); return r; } -export type ClaudePrivateRequest = { operation: "prepare-account"; credential: string; harnessExecutable: string; model: string } | { operation: "validate-saved"; agentId: string; connectionId: string; grantId: string }; +export type ClaudePrivateRequest = { operation: "check-harness"; harnessExecutable: string } | { operation: "prepare-account"; credential: string; harnessExecutable: string; model: string } | { operation: "validate-saved"; agentId: string; connectionId: string; grantId: string }; /** Only the selected per-attempt key/session crosses the private0600 file. * No Test request API, browser arguments, argv, env or raw child diagnostic gets it. */ export async function privateClaudeOperation(input: ClaudePrivateRequest & { companyId: string; environmentId: string; baseURL: string; cookies: Cookie[] }, scanSecret = input.operation === "prepare-account" ? input.credential : undefined) {