From 7244c70f11e0c6ed19a5a6d433cd1792c18cea37 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 00:28:33 -0500 Subject: [PATCH] test(evals): reconcile real account setup requests before resuming --- .../runtime-claude-account.test.ts | 19 +++++- .../runtime-claude-account.ts | 59 +++++++++++++++---- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.test.ts index 9539c9f1551..2714b79cdea 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeClaudeAccount, supportedClaudeSetupLink } from "./runtime-claude-account.js"; +import { gradeClaudeAccount, supportedClaudeSetupLink, completedClaudeSetupMatches } from "./runtime-claude-account.js"; function valid() { return { companyId: "company", leadId: "ceo", agents: [{ id: "morgan", name: "Morgan", companyId: "company", role: "engineer", title: "Frontend engineer", lifecycleState: "ready", status: "idle", adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "claude" } }], tasks: [], links: [], humanQueries: ["Hire Morgan as a frontend engineer using Claude Code.", "The Anthropic account is connected now. Continue the same Morgan frontend engineer hire using Claude Code."], reply: "", activity: [{ id: "hire", entityId: "morgan", action: "agent.hire_created", actorType: "agent", actorId: "ceo", runId: "second" }], runs: [{ id: "first", agentId: "ceo", status: "succeeded" }, { id: "second", agentId: "ceo", status: "succeeded" }], state: { initialAnthropicAccounts: 0, actualInstalledClaude: true, supportedSetupVisible: true, waitingReply: "Morgan's frontend hire is waiting for an Anthropic account. Connect the account using this setup request before Claude can start.", noUsableMorganBeforeHuman: true, accountOwnerIsHuman: true, humanAccountConnected: true, continuationMessageBound: true, beforeMorganId: "morgan", actualAccountBinding: true, savedValidation: { probeStatus: "pass", agentId: "morgan" }, continuationReply: "Morgan is hired and the selected account is connected.", initialRunId: "first", continuationRunId: "second" } }; } it("accepts preserved waiting hire followed by the actual human account and attributed continuation", () => { expect(gradeClaudeAccount(valid()).passed).toBe(true); const o = valid(); o.state.waitingReply = "I cannot finish the requested frontend hire until your Claude coding account is connected. Please use the supported setup flow."; o.state.continuationReply = "The frontend engineer has been added with the connected account."; expect(gradeClaudeAccount(o).passed).toBe(true); }); it("accepts a truthful future qualification instead of treating once-connected as current readiness", () => { const o = valid(); o.state.waitingReply = "Morgan is ready once the missing Anthropic account setup is complete; until then this hire is waiting for your connection."; expect(gradeClaudeAccount(o).passed).toBe(true); }); @@ -11,3 +11,20 @@ it("accepts same-resource account setup links and rejects foreign company/provid for (const href of ["/ACORN/apps/connect?source=anthropic", "http://fixture.invalid/ACORN/apps/connect?provider=claude", "/ACORN/company/settings/connections"]) expect(supportedClaudeSetupLink(href, "http://fixture.invalid", "ACORN")).toBe(true); for (const href of ["https://foreign.invalid/ACORN/apps/connect?source=anthropic", "/BIRCH/apps/connect?source=anthropic", "/ACORN/apps/connect?source=openai"]) expect(supportedClaudeSetupLink(href, "http://fixture.invalid", "ACORN")).toBe(false); }); + +it("requires real original setup-card reconciliation and exactly its declared automatic continuation", () => { + const value = valid() as ReturnType & { state: { setupRequestId?: string; setupRequestReconciled?: boolean; setupAutoWakeExpected?: boolean; setupAutoWakeBound?: boolean; autoMorganId?: string } }; + value.state.setupRequestId = "original-card"; + expect(gradeClaudeAccount(value).passed).toBe(false); + value.state.setupRequestReconciled = true; expect(gradeClaudeAccount(value).passed).toBe(true); + value.state.setupAutoWakeExpected = true; expect(gradeClaudeAccount(value).passed).toBe(false); + value.runs.push({ id: "automatic", agentId: "ceo", status: "succeeded" }); value.state.setupAutoWakeBound = true; value.state.autoMorganId = "morgan"; + expect(gradeClaudeAccount(value).passed).toBe(true); + value.state.setupAutoWakeBound = false; expect(gradeClaudeAccount(value).passed).toBe(false); +}); + +it("binds the resolved setup to the original actual human, source/native run and connected account DTO", () => { + const value = { sourceId: "source", sourceRunId: "first", companyId: "company", humanId: "person", connectionId: "account", request: { kind: "connection_intent", issueId: "source", companyId: "company", sourceRunId: "first", addresseeUserId: "person", status: "accepted", resolvedByUserId: "person", resolvedByAgentId: null, payload: { serviceSlug: "anthropic" }, result: { outcome: "connected", connectionId: "account" } } } as unknown as Parameters[0]; + expect(completedClaudeSetupMatches(value)).toBe(true); + for (const patch of [{ status: "pending" }, { sourceRunId: "wrong-run" }, { resolvedByUserId: "wrong-human" }, { resolvedByAgentId: "impersonating-agent" }, { issueId: "foreign-source" }, { result: { outcome: "connected", connectionId: "wrong-account" } }]) expect(completedClaudeSetupMatches({ ...value, request: { ...value.request!, ...patch } as typeof value.request })).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts index 527d851f7ea..747a8140129 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts @@ -1,5 +1,6 @@ import { expect } from "@playwright/test"; import type { Environment } from "../../../packages/shared/src/types/environment.js"; +import type { ConnectionIntentSetupOptions } from "../../../packages/shared/src/types/connection-intent.js"; import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js"; import type { Agent } from "../../../packages/shared/src/types/agent.js"; import type { ActivityEvent } from "../../../packages/shared/src/types/activity.js"; @@ -16,21 +17,26 @@ const continueHire = "The Anthropic account is connected now. Continue the same export function supportedClaudeSetupLink(href: string, origin: string, prefix: string) { try { const url = new URL(href, origin); return url.origin === origin && url.pathname.startsWith(`/${prefix}/`) && (url.pathname.endsWith("/company/settings/connections") || url.pathname.endsWith("/apps/connect") && ["anthropic", "claude"].includes(url.searchParams.get("source") ?? url.searchParams.get("provider") ?? "")); } catch { return false; } } +export function completedClaudeSetupMatches(input: { request: ConnectionIntentInteraction | undefined; sourceId: string; sourceRunId: string; companyId: string; humanId: string; connectionId: string }) { + const r = input.request; + return !!r && r.kind === "connection_intent" && r.issueId === input.sourceId && r.companyId === input.companyId && r.sourceRunId === input.sourceRunId && r.addresseeUserId === input.humanId + && r.payload.serviceSlug === "anthropic" && r.status === "accepted" && r.resolvedByUserId === input.humanId && !r.resolvedByAgentId && r.result?.outcome === "connected" && r.result.connectionId === input.connectionId; +} export function gradeClaudeAccount(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("genuine-missing-account-and-human-resumption-evidence", !!o, "Installed Claude, an actually absent eligible account, exact canonical waiting reply, real human connection and separately attributed continuation are required."); if (!o) return c.result(); const s = o.state, workers = o.agents.filter(row => row.name === "Morgan"), worker = workers[0] as (RuntimeAgent & Pick) | undefined, first = String(s.waitingReply), validation = s.savedValidation as ClaudePrivateReceipt | undefined; c.add("truthful-account-setup-with-original-hire-details", s.initialAnthropicAccounts === 0 && s.actualInstalledClaude === true && s.supportedSetupVisible === true && /Morgan|frontend|this hire|requested (?:hire|engineer)/i.test(first) && /Claude|Anthropic|coding|provider|model/i.test(first) && /account|connect|sign.?in|authenticat/i.test(first) && /need|wait|setup|set up|missing|before|blocked|to (?:finish|continue|complete)|requires|until|once|when/i.test(first) - && !/Morgan\s+(?:(?:is|now|already)\s+){1,2}(?:fully\s+)?ready\b(?!\s+(?:once|when|after|pending|only once|only after|subject to)\b)|\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:connected|authorized|authenticated)\b/i.test(first) && s.noUsableMorganBeforeHuman === true, + && !/Morgan\s+(?:(?:is|now|already)\s+){1,2}(?:fully\s+)?ready\b(?!\s+(?:once|when|after|pending|only once|only after|subject to)\b)|\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:connected|authorized|authenticated)\b/i.test(first) && s.noUsableMorganBeforeHuman === true && (!s.setupRequestId || s.setupRequestReconciled === true), "The first reply preserves the Morgan/Claude request and directs the responsible human to actual supported setup, without a ready hire or alternate harness before account access exists."); c.add("actual-human-account-followed-by-one-same-hire", s.accountOwnerIsHuman === true && s.humanAccountConnected === true && s.continuationMessageBound === true && workers.length === 1 && !!worker && worker.companyId === o.companyId && worker.lifecycleState === "ready" && ["idle", "active"].includes(worker.status ?? "") && worker.role === "engineer" && /frontend|front.end/i.test(`${worker.title ?? ""} ${worker.capabilities ?? ""}`) - && agentHarnessType(worker.adapterType ?? "", worker.adapterConfig) === "claude_local" && (!s.beforeMorganId || s.beforeMorganId === worker.id) + && agentHarnessType(worker.adapterType ?? "", worker.adapterConfig) === "claude_local" && (!s.beforeMorganId || s.beforeMorganId === worker.id) && (!s.autoMorganId || s.autoMorganId === worker.id) && s.actualAccountBinding === true && validation?.probeStatus === "pass" && validation.agentId === worker.id, "The signed-in human really connects the selected Anthropic account; the same preserved request then produces one usable permanent frontend Claude hire, preserving any initial pending identity."); c.add("phase-attributed-durable-reply-and-no-unrequested-work", /Morgan|frontend|engineer|hire/i.test(String(s.continuationReply)) && /creat|hir|saved|added|complete|ready|connected/i.test(String(s.continuationReply)) - && o.humanQueries.length === 2 && o.humanQueries[1] === continueHire && o.tasks.length === 0 && o.runs.length === 2 && o.runs.every(row => row.status === "succeeded" && row.agentId === o.leadId) + && o.humanQueries.length === 2 && o.humanQueries[1] === continueHire && o.tasks.length === 0 && o.runs.length === (s.setupAutoWakeExpected ? 3 : 2) && (!s.setupAutoWakeExpected || s.setupAutoWakeBound === true) && o.runs.every(row => row.status === "succeeded" && row.agentId === o.leadId) && o.activity.some(row => ["agent.created", "agent.hire_created"].includes(row.action ?? "") && row.entityId === worker?.id && row.actorId === o.leadId && row.actorType === "agent" && [s.initialRunId, s.continuationRunId].includes((row as unknown as ActivityEvent).runId)), "Only the two declared real human turns run; the second turn's actual native reply reports the saved hire and no employee work or duplicate agent is introduced."); return c.result(); @@ -45,8 +51,8 @@ export const runtimeClaudeAccountCases = [runtimeDefinition("agents.hire-needs-a await ctx.input.api.patch(`/api/companies/${companyId}`, { requireBoardApprovalForNewAgents: false }); return { selectedEnvironmentId: detail.id, initialAnthropicAccounts: 0, actualInstalledClaude: available.probeStatus === "pass" }; }, gradeClaudeAccount, { - ...runtimePrivateSpec("agents.hire-needs-account"), authentication: "human-session", expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 900000, - sourceFiles: ["chat-catalog-domains/runtime-claude-account.ts", "chat-catalog-domains/runtime-claude-private.ts", "chat-catalog-domains/runtime-claude-private-worker.ts", "chat-catalog-domains/runtime-private.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], + ...runtimePrivateSpec("agents.hire-needs-account"), authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 900000, + sourceFiles: ["chat-catalog-domains/runtime-claude-account.ts", "chat-catalog-domains/runtime-claude-private.ts", "chat-catalog-domains/runtime-claude-private-worker.ts", "chat-catalog-domains/runtime-private.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/routes/connection-intents.ts", "../../server/src/services/connection-intent-delivery.ts", "chat-scenarios/agents.yaml"], async act(ctx, state) { await waitForReply(ctx); const before = await observeRuntime(ctx, state), source = ctx.input.page.url(), firstRun = before.runs[0]!; state.initialRunId = firstRun.id; @@ -56,13 +62,14 @@ export const runtimeClaudeAccountCases = [runtimeDefinition("agents.hire-needs-a state.noUsableMorganBeforeHuman = pending.length <= 1 && pending.every(row => row.lifecycleState !== "ready" && agentHarnessType(row.adapterType ?? "", row.adapterConfig) === "claude_local"); const target = before.links.find(href => supportedClaudeSetupLink(href, before.origin, ctx.input.fixtures.company.issuePrefix!)); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + const requests = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`); + const cards = requests.filter(row => row.kind === "connection_intent" && row.sourceRunId === firstRun.id && row.payload.serviceSlug === "anthropic"); + if (cards.length > 1 || cards.some(row => row.status !== "pending" || row.addresseeUserId !== session.user.id)) throw new Error("The actual source setup request is duplicated, unavailable or addressed to a different human"); + const request = cards[0]; state.setupRequestId = request?.id; if (target) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(target)}]`).first().click(); else { - const requests = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`); - const request = requests.find(row => row.kind === "connection_intent" && row.status === "pending" && row.sourceRunId === firstRun.id && row.addresseeUserId === session.user.id && row.payload.serviceSlug === "anthropic"); if (!request) throw new Error("The canonical waiting response exposed neither a supported account setup link nor an actual addressed Anthropic setup request"); await ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(request.id)}]`).getByRole("button", { name: /Connect|Set up|Continue/i }).first().click(); - state.setupRequestId = request.id; } await expect(ctx.input.page.getByRole("heading", { name: /connections|Connect.*Anthropic|Connect a model provider/i }).first()).toBeVisible(); state.supportedSetupVisible = true; const selected = selectedRuntimeInputs(ctx, "agents.hire-needs-account"); @@ -72,14 +79,40 @@ export const runtimeClaudeAccountCases = [runtimeDefinition("agents.hire-needs-a state.accountOwnerIsHuman = saved?.ownerUserId === session.user.id; state.humanAccountConnected = saved?.status === "connected" && saved.provider === "anthropic" && account.probeStatus === "pass"; if (!state.humanAccountConnected || !state.accountOwnerIsHuman) throw new Error("The actual human account setup did not establish usable selected Anthropic access"); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/connections`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByText(saved!.name, { exact: true }).first()).toBeVisible(); - await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await sendChatMessage(ctx.input.page, continueHire); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + if (state.setupRequestId) { + const options = await ctx.input.api.get(`/api/connection-intents/${state.setupRequestId}/setup-options`); + if (options.interaction.id !== state.setupRequestId || options.interaction.issueId !== ctx.issue()!.id || options.interaction.companyId !== before.companyId || options.interaction.sourceRunId !== firstRun.id || options.interaction.addresseeUserId !== session.user.id) throw new Error("Account setup completion changed the addressed human/source request"); + const completion = options.aiConnectionRequiresAdoption + ? `/api/agents/${options.requestedAgentId}/connection-intents/${state.setupRequestId}/adopt?companyId=${encodeURIComponent(before.companyId)}` + : `/api/connection-intents/${state.setupRequestId}/complete`; + await ctx.input.api.post(completion, { connectionId: state.connectionId }); + const requests = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/interactions`), resolved = requests.find(row => row.id === state.setupRequestId); + state.setupRequestReconciled = completedClaudeSetupMatches({ request: resolved, sourceId: ctx.issue()!.id, sourceRunId: firstRun.id, companyId: before.companyId, humanId: session.user.id, connectionId: String(state.connectionId) }); + if (!state.setupRequestReconciled) throw new Error("The actual addressed Anthropic setup request did not reconcile against established human account access"); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(String(state.setupRequestId))}]`)).toContainText(/connected|ready|complete/i); + await ctx.refreshIssue(); + state.setupAutoWakeExpected = ["in_progress", "in_review"].includes(ctx.issue()!.status) && ctx.issue()!.assigneeAgentId === resolved!.payload.requestingAgentId; + if (state.setupAutoWakeExpected) { + const automatic = await pollUntil({ label: "Actual accepted setup request delivers its declared automatic native continuation", deadlineAt: Date.now() + 180000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.every(row => row.status === "succeeded"), + reject: v => v.runs.length > 2 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "The actual setup-resolution automatic continuation failed or duplicated" : undefined }); + const run = automatic.runs.find(row => row.id !== firstRun.id)!; + state.setupAutoWakeBound = run.contextSnapshot?.source === "connection_intent.resolved" && run.contextSnapshot.interactionId === state.setupRequestId && run.contextSnapshot.interactionStatus === "accepted" && run.contextSnapshot.issueId === ctx.issue()!.id; + const agents = await ctx.input.api.get(`/api/companies/${before.companyId}/agents`); state.autoMorganId = agents.find(row => row.name === "Morgan")?.id; + } else if ((await ctx.allRuns()).length !== 1) throw new Error("An ineligible setup completion admitted undeclared native work"); + } + const priorConfirmationIds = (await ctx.allRuns()).map(row => row.id); + await sendChatMessage(ctx.input.page, continueHire); const settled = await pollUntil({ label: "Actual explicit human account confirmation resumes the preserved Claude hire", deadlineAt: Date.now() + 240000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns(), agents: await ctx.input.api.get(`/api/companies/${before.companyId}/agents`) }; }, - accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.every(row => row.status === "succeeded") && v.agents.filter(row => row.name === "Morgan").length === 1 && v.agents.find(row => row.name === "Morgan")?.lifecycleState === "ready", - reject: v => v.runs.length > 2 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Account resumption failed or admitted undeclared duplicate work" : undefined }); - const run = settled.runs.find(row => row.id !== firstRun.id)!, next = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`); + accept: v => v.source?.conversationState === "waiting" && v.runs.length === priorConfirmationIds.length + 1 && v.runs.every(row => row.status === "succeeded") && v.agents.filter(row => row.name === "Morgan").length === 1 && v.agents.find(row => row.name === "Morgan")?.lifecycleState === "ready", + reject: v => v.runs.length > priorConfirmationIds.length + 1 || v.runs.some(row => ["failed", "cancelled", "timed_out"].includes(row.status)) ? "Account resumption failed or admitted undeclared duplicate work" : undefined }); + const run = settled.runs.find(row => !priorConfirmationIds.includes(row.id))!, next = await ctx.input.api.get<{ contextSnapshot?: { wakeCommentIds?: string[] }; runnerProfileJson?: { nativeExecutionInput?: { task?: { prompt?: string } } } }>(`/api/heartbeat-runs/${run.id}`); const after = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), human = after.filter(row => !row.authorAgentId && row.body === continueHire); - state.continuationMessageBound = human.length === 1 && next.contextSnapshot?.wakeCommentIds?.length === 1 && next.contextSnapshot.wakeCommentIds[0] === human[0]?.id && next.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(continueHire) === true; + state.continuationMessageBound = human.length === 1 && human[0]?.authorUserId === session.user.id && next.contextSnapshot?.wakeCommentIds?.length === 1 && next.contextSnapshot.wakeCommentIds[0] === human[0]?.id && next.runnerProfileJson?.nativeExecutionInput?.task?.prompt?.includes(continueHire) === true; state.continuationRunId = run.id; state.continuationReply = after.filter(row => row.authorAgentId === before.leadId && row.createdByRunId === run.id).map(row => row.body).join("\n"); const worker = settled.agents.find(row => row.name === "Morgan")!; state.worker = worker; state.savedValidation = await privateClaudeOperation({ operation: "validate-saved", companyId: before.companyId, environmentId: String(state.selectedEnvironmentId), agentId: worker.id, connectionId: String(state.connectionId), grantId: String(state.grantId), baseURL: ctx.input.api.baseURL, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }, selected.credentials.ANTHROPIC_API_KEY);