From 0d43238baee78fca6c257873a5a3e6a32b3edf42 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 6 Oct 2026 13:29:40 -0500 Subject: [PATCH] fix(setup): keep sign-in environment selection and cleanup evidence reachable Co-Authored-By: Paperclip --- tests/runner-e2e/PROVIDER-CONNECTIONS.md | 2 + tests/runner-e2e/connection-launch.ts | 74 +++++++++++++------ tests/runner-e2e/connection-support.test.ts | 44 ++++++++++- ui/src/components/new-agent/NewAgentSetup.tsx | 15 +++- ui/src/pages/NewAgent.test.tsx | 29 ++++++++ 5 files changed, 136 insertions(+), 28 deletions(-) diff --git a/tests/runner-e2e/PROVIDER-CONNECTIONS.md b/tests/runner-e2e/PROVIDER-CONNECTIONS.md index 32f022fc8aa..22b5a94135f 100644 --- a/tests/runner-e2e/PROVIDER-CONNECTIONS.md +++ b/tests/runner-e2e/PROVIDER-CONNECTIONS.md @@ -196,6 +196,8 @@ file-conversion, session-configuration, or shell-restriction failures. It reads error records and failed tool receipts only; messages, stderr, model output, and reasoning are not copied into the result. An empty signal list leaves the cause unknown. Missing logs remain explicit and do not imply a healthy run. +Attached-company cleanup captures these diagnostics after stopping runs and +before deleting the fixture agent, since agent deletion also removes its logs. UI probes may incur unreported spend, so billing is explicitly partial. Regenerate the report from retained results with `pnpm test:e2e:runner:dashboard -- /absolute/path/to/connections-campaign`. diff --git a/tests/runner-e2e/connection-launch.ts b/tests/runner-e2e/connection-launch.ts index f85d52a75ee..2b59ea6cd55 100644 --- a/tests/runner-e2e/connection-launch.ts +++ b/tests/runner-e2e/connection-launch.ts @@ -21,6 +21,40 @@ import type { MatrixExecution, RunnerE2EResult } from "./types.js"; type Row = Record; +export async function cleanupConnectionCompany(input: { + api: ConnectionApi; companyId: string; agentName: string; + attachedCompany: boolean; retainCompany: boolean; + evidence: Pick; + collectDiagnostics: () => Promise; +}) { + const { api, companyId, agentName, evidence } = input; + const agents = await api.get(`/api/companies/${companyId}/agents`); + const fixtureAgents = agents.filter(agent => agent.name === agentName); + try { + for (const agent of fixtureAgents) { + await api.patch(`/api/agents/${agent.id}`, { status: "paused" }); + const liveRuns = await api.get(`/api/companies/${companyId}/heartbeat-runs?agentId=${agent.id}&limit=100`); + for (const run of liveRuns.filter(run => run.agentId === agent.id && ["queued", "running"].includes(run.status))) { + await api.post(`/api/heartbeat-runs/${run.id}/cancel`); + } + } + } finally { + // Deleting an attached-company agent also deletes its runs and logs. + // Capture the stopped runs first, including when cancellation fails. + await input.collectDiagnostics(); + } + if (input.retainCompany) { evidence.cleanupRetained = true; return; } + const connections = (await api.get(`/api/companies/${companyId}/ai-connections`)).connections; + for (const connection of connections.filter((connection: Row) => connection.status !== "revoked")) { + await api.delete(`/api/tool-connections/${connection.id}`); + } + if ((await api.get(`/api/companies/${companyId}/ai-connections`)).connections.some((connection: Row) => connection.status !== "revoked")) { + throw new ConnectionFailure("connection_revocation_failed"); + } + if (!input.attachedCompany) { await api.post(`/api/companies/${companyId}/archive`); evidence.companyArchived = true; } + else for (const agent of fixtureAgents) await api.delete(`/api/agents/${agent.id}`); +} + export async function openConnectionBrowser(config: ConnectionConfig, repositoryRoot: string, origin: string) { const temporary = config.browser.freshness === "signed-out" || !config.browser.profileDir; const profile = temporary ? await mkdtemp(path.join(os.tmpdir(), "paperclip-qa-browser-")) : path.resolve(config.browser.profileDir!); @@ -95,6 +129,18 @@ async function connectionAttempt(execution: MatrixExecution, config: ConnectionC let fixtures: Awaited> | undefined; const observedRuns = new Map(); let diagnosticApi: ConnectionApi | undefined; + let diagnosticsCollected = false; + const collectDiagnostics = async () => { + if (diagnosticsCollected || !diagnosticApi || !observedRuns.size) return; + evidence.runDiagnostics = []; + for (const observed of observedRuns.values()) { + let run = observed; let log: unknown; + try { run = await diagnosticApi.get(`/api/heartbeat-runs/${observed.id}`); observedRuns.set(observed.id, run); } catch { /* Keep the last durable observation. */ } + try { log = await diagnosticApi.get(`/api/heartbeat-runs/${observed.id}/log?limitBytes=2000000`); } catch { /* Mark missing log evidence explicitly. */ } + evidence.runDiagnostics.push(connectionRunDiagnostic(run, log)); + } + diagnosticsCollected = true; + }; const loginSessions = new Map(); const loginObservers: Promise[] = []; let stopped: "test_interrupted" | "cell_deadline_reached" | undefined; @@ -164,21 +210,10 @@ async function connectionAttempt(execution: MatrixExecution, config: ConnectionC try { await api.response(route, method); } catch (error) { if (!(error instanceof ConnectionFailure) || error.message !== "api_status_404") throw error; } } - const agents = await api.get(`/api/companies/${company.id}/agents`); - for (const agent of agents.filter(agent => agent.name === `Connection QA ${nonce}`)) { - await api.patch(`/api/agents/${agent.id}`, { status: "paused" }); - const liveRuns = await api.get(`/api/companies/${company.id}/heartbeat-runs?agentId=${agent.id}&limit=100`); - const active = liveRuns.filter(run => run.agentId === agent.id && ["queued", "running"].includes(run.status)); - for (const run of active) await api.post(`/api/heartbeat-runs/${run.id}/cancel`); - } - if (config.retainCompany) { evidence.cleanupRetained = true; return; } // The fixture verified the company was empty before running. Revoke all // newly created test connections even if a later UI assertion failed. - const connections = (await api.get(`/api/companies/${company.id}/ai-connections`)).connections; - for (const connection of connections.filter((connection: Row) => connection.status !== "revoked")) await api.delete(`/api/tool-connections/${connection.id}`); - if ((await api.get(`/api/companies/${company.id}/ai-connections`)).connections.some((connection: Row) => connection.status !== "revoked")) throw new ConnectionFailure("connection_revocation_failed"); - if (!attachedCompany) { await api.post(`/api/companies/${company.id}/archive`); evidence.companyArchived = true; } - else for (const agent of agents.filter(agent => agent.name === `Connection QA ${nonce}`)) await api.delete(`/api/agents/${agent.id}`); + await cleanupConnectionCompany({ api, companyId: company.id, agentName: `Connection QA ${nonce}`, + attachedCompany: Boolean(attachedCompany), retainCompany: config.retainCompany, evidence, collectDiagnostics }); } }); registry.register({ id: "connection-environment", dependencies: ["connection-company"], setup: async values => { const company = values.get("connection-company") as Row; @@ -222,16 +257,9 @@ async function connectionAttempt(execution: MatrixExecution, config: ConnectionC catch (cleanupError) { result.cleanup = "failed"; result.status = "failed"; result.failureClass = "cleanup_failure"; result.error = `${result.error ? `${result.error}; ` : ""}fixture_cleanup_failed`; const causes = cleanupError instanceof AggregateError ? cleanupError.errors : [cleanupError]; console.log(`[provider-connections] Cleanup: ${causes.map(cause => safeConnectionFailure(cause, "cleanup")).join(", ")}`); } - // Project errors before a managed target removes its DB/logs. - if (diagnosticApi && observedRuns.size) { - evidence.runDiagnostics = []; - for (const observed of observedRuns.values()) { - let run = observed; let log: unknown; - try { run = await diagnosticApi.get(`/api/heartbeat-runs/${observed.id}`); observedRuns.set(observed.id, run); } catch { /* Keep the last durable observation. */ } - try { log = await diagnosticApi.get(`/api/heartbeat-runs/${observed.id}/log?limitBytes=2000000`); } catch { /* Mark missing log evidence explicitly. */ } - evidence.runDiagnostics.push(connectionRunDiagnostic(run, log)); - } - } + // Also preserve evidence when setup or early cleanup failed before the + // company teardown reached its capture point. Never refetch deleted runs. + await collectDiagnostics(); try { await browser?.close(); } catch { result.cleanup = "failed"; result.status = "failed"; result.failureClass = "cleanup_failure"; result.error = "qa_browser_cleanup_failed"; } result.finishedAt = new Date().toISOString(); diff --git a/tests/runner-e2e/connection-support.test.ts b/tests/runner-e2e/connection-support.test.ts index f52bfc7d969..c39416c31ba 100644 --- a/tests/runner-e2e/connection-support.test.ts +++ b/tests/runner-e2e/connection-support.test.ts @@ -10,7 +10,8 @@ import { parseConnectionConfig, targetOrigin, selectedSecret, resolveConnectionS import { connectionCheckpoints, connectionEvidencePasses, connectionRunDiagnostic, createConnectionProof, verifyConnectionArtifact, verifyConnectionRun, completedConnectionArtifactRun, connectionProbeChecks, type ConnectionEvidence } from "./connection-evidence.js"; import { validateRetainedRunnerResult } from "./result-validation.js"; import { verifyConnectionTarget } from "./connection-target.js"; -import { runConnectionCampaign, safeConnectionFailure, withConnectionCampaignCancellation } from "./connection-launch.js"; +import { cleanupConnectionCompany, runConnectionCampaign, safeConnectionFailure, withConnectionCampaignCancellation } from "./connection-launch.js"; +import type { ConnectionApi } from "./connection-target.js"; import * as connectionConfig from "./connection-config.js"; import * as connectionTarget from "./connection-target.js"; import * as reportAssets from "./report-assets.js"; @@ -21,6 +22,47 @@ import type { RunnerE2EResult } from "./types.js"; const cells = runnerMatrix.filter(e => e.suite.id === "provider-connections"); const cell = (method: string) => cells.find(e => e.id === `provider-connections.connection-codex-native.local.agent-${method}`)!; +describe("connection fixture cleanup evidence", () => { + it.each([true, false])("captures stopped runs and logs before destructive cleanup (attached=%s)", async (attachedCompany) => { + const events: string[] = []; + let status = "running", deleted = false, revoked = false; + const evidence: Partial = {}; + const api = { + get: vi.fn(async (route: string) => { + if (route.endsWith("/agents")) return [{ id: "qa-agent", name: "Fixture" }, { id: "foreign", name: "Other" }]; + if (route.includes("heartbeat-runs?")) return [{ id: "run", agentId: "qa-agent", status }]; + if (route.endsWith("/ai-connections")) return { connections: [{ id: "account", status: revoked ? "revoked" : "active" }] }; + if (deleted) throw new Error("Run was deleted"); + if (route.includes("/log?")) { events.push("log"); return { content: JSON.stringify({ chunk: JSON.stringify({ type: "acpx.error", childStderrTail: "HTTP 503 model overloaded" }) + "\n" }) }; } + events.push("run"); return { id: "run", status }; + }), + patch: vi.fn(async () => { events.push("pause"); }), + post: vi.fn(async (route: string) => { if (route.endsWith("/cancel")) { events.push("cancel"); status = "cancelled"; } else events.push("archive"); }), + delete: vi.fn(async (route: string) => { if (route.includes("tool-connections")) { events.push("revoke"); revoked = true; } else { events.push("delete-agent"); deleted = true; } }), + }; + await cleanupConnectionCompany({ api: api as unknown as ConnectionApi, companyId: "company", agentName: "Fixture", attachedCompany, + retainCompany: false, evidence, collectDiagnostics: async () => { + const run = await api.get("/api/heartbeat-runs/run"); + const log = await api.get("/api/heartbeat-runs/run/log?limitBytes=2000000"); + evidence.runDiagnostics = [connectionRunDiagnostic(run, log)]; + } }); + expect(events).toEqual(["pause", "cancel", "run", "log", "revoke", attachedCompany ? "delete-agent" : "archive"]); + expect(evidence.runDiagnostics).toEqual([{ runId: "run", status: "cancelled", signals: ["provider_overloaded"], logAvailable: true }]); + expect(api.delete).not.toHaveBeenCalledWith("/api/agents/foreign"); + }); + it("preserves diagnostics and avoids deletion when cancellation fails", async () => { + const collectDiagnostics = vi.fn(async () => {}); + const api = { + get: vi.fn(async (route: string) => route.endsWith("/agents") ? [{ id: "qa-agent", name: "Fixture" }] : [{ id: "run", agentId: "qa-agent", status: "running" }]), + patch: vi.fn(async () => {}), post: vi.fn(async () => { throw new Error("cancel failed"); }), delete: vi.fn(), + }; + await expect(cleanupConnectionCompany({ api: api as unknown as ConnectionApi, companyId: "company", agentName: "Fixture", attachedCompany: true, + retainCompany: false, evidence: {}, collectDiagnostics })).rejects.toThrow("cancel failed"); + expect(collectDiagnostics).toHaveBeenCalledOnce(); + expect(api.delete).not.toHaveBeenCalled(); + }); +}); + describe("campaign interruption", () => { it.each(["SIGINT", "SIGTERM", "SIGHUP"] as const)("keeps %s cancellation active until teardown finishes", async (signal) => { const before = process.listenerCount(signal); diff --git a/ui/src/components/new-agent/NewAgentSetup.tsx b/ui/src/components/new-agent/NewAgentSetup.tsx index e2e5819ee2b..1bfa36d7b82 100644 --- a/ui/src/components/new-agent/NewAgentSetup.tsx +++ b/ui/src/components/new-agent/NewAgentSetup.tsx @@ -684,7 +684,7 @@ function Setup({ type="button" aria-current={step === screen ? "step" : undefined} disabled={ - busy || Boolean(created) || index > steps.indexOf(screen) + busy || Boolean(created) || step === "saved" } onClick={() => setScreen(step)} className={cn( @@ -1143,7 +1143,7 @@ function Setup({ state={testState} result={result} error={error} - disabled={!ready || busy} + disabled={!ready || busy || Boolean(connectionAdapter && !connection)} onTest={() => void runTest()} /> {error && testState !== "fail" && ( @@ -1165,7 +1165,14 @@ function Setup({ ) : ( )} - : + } )} diff --git a/ui/src/pages/NewAgent.test.tsx b/ui/src/pages/NewAgent.test.tsx index 50ae95309a0..e31a20ebb06 100644 --- a/ui/src/pages/NewAgent.test.tsx +++ b/ui/src/pages/NewAgent.test.tsx @@ -200,6 +200,35 @@ afterEach(async () => { container.remove(); }); describe("New agent setup", () => { + it.each(["claude_local", "codex_local"])("can choose a sign-in environment in Configure before connecting %s", async (adapterType) => { + settings.get.mockResolvedValue({ defaultEnvironmentId: "no-login" }); + envApi.list.mockResolvedValue([ + { id: "no-login", name: "No browser sign-in", status: "active", driver: "sandbox", config: { provider: "other" } }, + { id: "login-env", name: "Browser sign-in", status: "active", driver: "sandbox", config: { provider: "daytona" } }, + ]); + envApi.capabilities.mockResolvedValue({ sandboxProviders: { daytona: { supportsLoginPty: true } } }); + api.getAdapterAuthSignal.mockResolvedValue({ status: "missing" }); + await render(adapterType); + expect(container.textContent).toContain("does not support browser sign-in"); + expect(container.querySelector('select[aria-label="Environment"]')).toBeNull(); + await click("2Configure"); + expect(container.querySelector('button[type="submit"]')).toBeNull(); + expect([...container.querySelectorAll("button")].find(button => button.textContent?.trim() === "Run test")?.disabled).toBe(true); + await click("Connect model"); + expect(api.testEnvironment).not.toHaveBeenCalled(); + expect(api.hire).not.toHaveBeenCalled(); + await click("2Configure"); + const select = container.querySelector('select[aria-label="Environment"]') as HTMLSelectElement; + await act(async () => { + select.value = "login-env"; + select.dispatchEvent(new Event("change", { bubbles: true })); + }); + await settle(); + expect(container.querySelector('select[aria-label="Environment"]')).toBeNull(); + await click("Complete subscription login"); + await click("Finish setup"); + expect(api.hire.mock.calls[0][1]).toMatchObject({ defaultEnvironmentId: "login-env", runtimeConfig: { aiConnection: { method: "subscription" } } }); + }); it("waits for the resolved environment before allowing a credential-mode selection", async () => { let resolveEnvironments!: (value: object[]) => void; envApi.list.mockImplementationOnce(() => new Promise(resolve => { resolveEnvironments = resolve; }));