fix(setup): keep sign-in environment selection and cleanup evidence reachable

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/PROVIDER-CONNECTIONS.md

@@ -196,6 +196,8 @@ file-conversion, session-configuration, or shell-restriction failures. It reads
 error records and failed tool receipts only; messages, stderr, model output,
 and reasoning are not copied into the result. An empty signal list leaves the
 cause unknown. Missing logs remain explicit and do not imply a healthy run.
+Attached-company cleanup captures these diagnostics after stopping runs and
+before deleting the fixture agent, since agent deletion also removes its logs.
 UI probes may incur unreported spend, so billing is explicitly partial.
 Regenerate the report from retained results with
 `pnpm test:e2e:runner:dashboard -- /absolute/path/to/connections-campaign`.

tests/runner-e2e/connection-launch.ts

@@ -21,6 +21,40 @@ import type { MatrixExecution, RunnerE2EResult } from "./types.js";
 
 type Row = Record<string, any>;
 
+export async function cleanupConnectionCompany(input: {
+  api: ConnectionApi; companyId: string; agentName: string;
+  attachedCompany: boolean; retainCompany: boolean;
+  evidence: Pick<ConnectionEvidence, "cleanupRetained" | "companyArchived">;
+  collectDiagnostics: () => Promise<void>;
+}) {
+  const { api, companyId, agentName, evidence } = input;
+  const agents = await api.get<Row[]>(`/api/companies/${companyId}/agents`);
+  const fixtureAgents = agents.filter(agent => agent.name === agentName);
+  try {
+    for (const agent of fixtureAgents) {
+      await api.patch(`/api/agents/${agent.id}`, { status: "paused" });
+      const liveRuns = await api.get<Row[]>(`/api/companies/${companyId}/heartbeat-runs?agentId=${agent.id}&limit=100`);
+      for (const run of liveRuns.filter(run => run.agentId === agent.id && ["queued", "running"].includes(run.status))) {
+        await api.post(`/api/heartbeat-runs/${run.id}/cancel`);
+      }
+    }
+  } finally {
+    // Deleting an attached-company agent also deletes its runs and logs.
+    // Capture the stopped runs first, including when cancellation fails.
+    await input.collectDiagnostics();
+  }
+  if (input.retainCompany) { evidence.cleanupRetained = true; return; }
+  const connections = (await api.get(`/api/companies/${companyId}/ai-connections`)).connections;
+  for (const connection of connections.filter((connection: Row) => connection.status !== "revoked")) {
+    await api.delete(`/api/tool-connections/${connection.id}`);
+  }
+  if ((await api.get(`/api/companies/${companyId}/ai-connections`)).connections.some((connection: Row) => connection.status !== "revoked")) {
+    throw new ConnectionFailure("connection_revocation_failed");
+  }
+  if (!input.attachedCompany) { await api.post(`/api/companies/${companyId}/archive`); evidence.companyArchived = true; }
+  else for (const agent of fixtureAgents) await api.delete(`/api/agents/${agent.id}`);
+}
+
 export async function openConnectionBrowser(config: ConnectionConfig, repositoryRoot: string, origin: string) {
   const temporary = config.browser.freshness === "signed-out" || !config.browser.profileDir;
   const profile = temporary ? await mkdtemp(path.join(os.tmpdir(), "paperclip-qa-browser-")) : path.resolve(config.browser.profileDir!);
@@ -95,6 +129,18 @@ async function connectionAttempt(execution: MatrixExecution, config: ConnectionC
   let fixtures: Awaited<ReturnType<FixtureRegistry["setupAll"]>> | undefined;
   const observedRuns = new Map<string, Row>();
   let diagnosticApi: ConnectionApi | undefined;
+  let diagnosticsCollected = false;
+  const collectDiagnostics = async () => {
+    if (diagnosticsCollected || !diagnosticApi || !observedRuns.size) return;
+    evidence.runDiagnostics = [];
+    for (const observed of observedRuns.values()) {
+      let run = observed; let log: unknown;
+      try { run = await diagnosticApi.get(`/api/heartbeat-runs/${observed.id}`); observedRuns.set(observed.id, run); } catch { /* Keep the last durable observation. */ }
+      try { log = await diagnosticApi.get(`/api/heartbeat-runs/${observed.id}/log?limitBytes=2000000`); } catch { /* Mark missing log evidence explicitly. */ }
+      evidence.runDiagnostics.push(connectionRunDiagnostic(run, log));
+    }
+    diagnosticsCollected = true;
+  };
   const loginSessions = new Map<string, "DELETE" | "POST">();
   const loginObservers: Promise<void>[] = [];
   let stopped: "test_interrupted" | "cell_deadline_reached" | undefined;
@@ -164,21 +210,10 @@ async function connectionAttempt(execution: MatrixExecution, config: ConnectionC
         try { await api.response(route, method); }
         catch (error) { if (!(error instanceof ConnectionFailure) || error.message !== "api_status_404") throw error; }
       }
-      const agents = await api.get<Row[]>(`/api/companies/${company.id}/agents`);
-      for (const agent of agents.filter(agent => agent.name === `Connection QA ${nonce}`)) {
-        await api.patch(`/api/agents/${agent.id}`, { status: "paused" });
-        const liveRuns = await api.get<Row[]>(`/api/companies/${company.id}/heartbeat-runs?agentId=${agent.id}&limit=100`);
-        const active = liveRuns.filter(run => run.agentId === agent.id && ["queued", "running"].includes(run.status));
-        for (const run of active) await api.post(`/api/heartbeat-runs/${run.id}/cancel`);
-      }
-      if (config.retainCompany) { evidence.cleanupRetained = true; return; }
       // The fixture verified the company was empty before running. Revoke all
       // newly created test connections even if a later UI assertion failed.
-      const connections = (await api.get(`/api/companies/${company.id}/ai-connections`)).connections;
-      for (const connection of connections.filter((connection: Row) => connection.status !== "revoked")) await api.delete(`/api/tool-connections/${connection.id}`);
-      if ((await api.get(`/api/companies/${company.id}/ai-connections`)).connections.some((connection: Row) => connection.status !== "revoked")) throw new ConnectionFailure("connection_revocation_failed");
-      if (!attachedCompany) { await api.post(`/api/companies/${company.id}/archive`); evidence.companyArchived = true; }
-      else for (const agent of agents.filter(agent => agent.name === `Connection QA ${nonce}`)) await api.delete(`/api/agents/${agent.id}`);
+      await cleanupConnectionCompany({ api, companyId: company.id, agentName: `Connection QA ${nonce}`,
+        attachedCompany: Boolean(attachedCompany), retainCompany: config.retainCompany, evidence, collectDiagnostics });
     } });
     registry.register<Row>({ id: "connection-environment", dependencies: ["connection-company"], setup: async values => {
       const company = values.get("connection-company") as Row;
@@ -222,16 +257,9 @@ async function connectionAttempt(execution: MatrixExecution, config: ConnectionC
     catch (cleanupError) { result.cleanup = "failed"; result.status = "failed"; result.failureClass = "cleanup_failure"; result.error = `${result.error ? `${result.error}; ` : ""}fixture_cleanup_failed`;
       const causes = cleanupError instanceof AggregateError ? cleanupError.errors : [cleanupError];
       console.log(`[provider-connections] Cleanup: ${causes.map(cause => safeConnectionFailure(cause, "cleanup")).join(", ")}`); }
-    // Project errors before a managed target removes its DB/logs.
-    if (diagnosticApi && observedRuns.size) {
-      evidence.runDiagnostics = [];
-      for (const observed of observedRuns.values()) {
-        let run = observed; let log: unknown;
-        try { run = await diagnosticApi.get(`/api/heartbeat-runs/${observed.id}`); observedRuns.set(observed.id, run); } catch { /* Keep the last durable observation. */ }
-        try { log = await diagnosticApi.get(`/api/heartbeat-runs/${observed.id}/log?limitBytes=2000000`); } catch { /* Mark missing log evidence explicitly. */ }
-        evidence.runDiagnostics.push(connectionRunDiagnostic(run, log));
-      }
-    }
+    // Also preserve evidence when setup or early cleanup failed before the
+    // company teardown reached its capture point. Never refetch deleted runs.
+    await collectDiagnostics();
     try { await browser?.close(); }
     catch { result.cleanup = "failed"; result.status = "failed"; result.failureClass = "cleanup_failure"; result.error = "qa_browser_cleanup_failed"; }
     result.finishedAt = new Date().toISOString();

tests/runner-e2e/connection-support.test.ts

@@ -10,7 +10,8 @@ import { parseConnectionConfig, targetOrigin, selectedSecret, resolveConnectionS
 import { connectionCheckpoints, connectionEvidencePasses, connectionRunDiagnostic, createConnectionProof, verifyConnectionArtifact, verifyConnectionRun, completedConnectionArtifactRun, connectionProbeChecks, type ConnectionEvidence } from "./connection-evidence.js";
 import { validateRetainedRunnerResult } from "./result-validation.js";
 import { verifyConnectionTarget } from "./connection-target.js";
-import { runConnectionCampaign, safeConnectionFailure, withConnectionCampaignCancellation } from "./connection-launch.js";
+import { cleanupConnectionCompany, runConnectionCampaign, safeConnectionFailure, withConnectionCampaignCancellation } from "./connection-launch.js";
+import type { ConnectionApi } from "./connection-target.js";
 import * as connectionConfig from "./connection-config.js";
 import * as connectionTarget from "./connection-target.js";
 import * as reportAssets from "./report-assets.js";
@@ -21,6 +22,47 @@ import type { RunnerE2EResult } from "./types.js";
 const cells = runnerMatrix.filter(e => e.suite.id === "provider-connections");
 const cell = (method: string) => cells.find(e => e.id === `provider-connections.connection-codex-native.local.agent-${method}`)!;
 
+describe("connection fixture cleanup evidence", () => {
+  it.each([true, false])("captures stopped runs and logs before destructive cleanup (attached=%s)", async (attachedCompany) => {
+    const events: string[] = [];
+    let status = "running", deleted = false, revoked = false;
+    const evidence: Partial<ConnectionEvidence> = {};
+    const api = {
+      get: vi.fn(async (route: string) => {
+        if (route.endsWith("/agents")) return [{ id: "qa-agent", name: "Fixture" }, { id: "foreign", name: "Other" }];
+        if (route.includes("heartbeat-runs?")) return [{ id: "run", agentId: "qa-agent", status }];
+        if (route.endsWith("/ai-connections")) return { connections: [{ id: "account", status: revoked ? "revoked" : "active" }] };
+        if (deleted) throw new Error("Run was deleted");
+        if (route.includes("/log?")) { events.push("log"); return { content: JSON.stringify({ chunk: JSON.stringify({ type: "acpx.error", childStderrTail: "HTTP 503 model overloaded" }) + "\n" }) }; }
+        events.push("run"); return { id: "run", status };
+      }),
+      patch: vi.fn(async () => { events.push("pause"); }),
+      post: vi.fn(async (route: string) => { if (route.endsWith("/cancel")) { events.push("cancel"); status = "cancelled"; } else events.push("archive"); }),
+      delete: vi.fn(async (route: string) => { if (route.includes("tool-connections")) { events.push("revoke"); revoked = true; } else { events.push("delete-agent"); deleted = true; } }),
+    };
+    await cleanupConnectionCompany({ api: api as unknown as ConnectionApi, companyId: "company", agentName: "Fixture", attachedCompany,
+      retainCompany: false, evidence, collectDiagnostics: async () => {
+        const run = await api.get("/api/heartbeat-runs/run");
+        const log = await api.get("/api/heartbeat-runs/run/log?limitBytes=2000000");
+        evidence.runDiagnostics = [connectionRunDiagnostic(run, log)];
+      } });
+    expect(events).toEqual(["pause", "cancel", "run", "log", "revoke", attachedCompany ? "delete-agent" : "archive"]);
+    expect(evidence.runDiagnostics).toEqual([{ runId: "run", status: "cancelled", signals: ["provider_overloaded"], logAvailable: true }]);
+    expect(api.delete).not.toHaveBeenCalledWith("/api/agents/foreign");
+  });
+  it("preserves diagnostics and avoids deletion when cancellation fails", async () => {
+    const collectDiagnostics = vi.fn(async () => {});
+    const api = {
+      get: vi.fn(async (route: string) => route.endsWith("/agents") ? [{ id: "qa-agent", name: "Fixture" }] : [{ id: "run", agentId: "qa-agent", status: "running" }]),
+      patch: vi.fn(async () => {}), post: vi.fn(async () => { throw new Error("cancel failed"); }), delete: vi.fn(),
+    };
+    await expect(cleanupConnectionCompany({ api: api as unknown as ConnectionApi, companyId: "company", agentName: "Fixture", attachedCompany: true,
+      retainCompany: false, evidence: {}, collectDiagnostics })).rejects.toThrow("cancel failed");
+    expect(collectDiagnostics).toHaveBeenCalledOnce();
+    expect(api.delete).not.toHaveBeenCalled();
+  });
+});
+
 describe("campaign interruption", () => {
   it.each(["SIGINT", "SIGTERM", "SIGHUP"] as const)("keeps %s cancellation active until teardown finishes", async (signal) => {
     const before = process.listenerCount(signal);

ui/src/components/new-agent/NewAgentSetup.tsx

@@ -684,7 +684,7 @@ function Setup({
                     type="button"
                     aria-current={step === screen ? "step" : undefined}
                     disabled={
-                      busy || Boolean(created) || index > steps.indexOf(screen)
+                      busy || Boolean(created) || step === "saved"
                     }
                     onClick={() => setScreen(step)}
                     className={cn(
@@ -1143,7 +1143,7 @@ function Setup({
                       state={testState}
                       result={result}
                       error={error}
-                      disabled={!ready || busy}
+                      disabled={!ready || busy || Boolean(connectionAdapter && !connection)}
                       onTest={() => void runTest()}
                     />
                     {error && testState !== "fail" && (
@@ -1165,7 +1165,14 @@ function Setup({
                       ) : (
                         <span />
                       )}
-                      <Button
+                      {connectionAdapter && !connection ? <Button
+                        type="button"
+                        disabled={!ready || busy}
+                        onClick={() => setScreen("connect")}
+                      >
+                        Connect model
+                        <ArrowRight className="size-4" />
+                      </Button> : <Button
                         type="submit"
                         disabled={
                           !ready ||
@@ -1176,7 +1183,7 @@ function Setup({
                       >
                         {saving ? "Creating…" : "Finish setup"}
                         <Check className="size-4" />
-                      </Button>
+                      </Button>}
                     </div>
                   </form>
                 )}

ui/src/pages/NewAgent.test.tsx

@@ -200,6 +200,35 @@ afterEach(async () => {
   container.remove();
 });
 describe("New agent setup", () => {
+  it.each(["claude_local", "codex_local"])("can choose a sign-in environment in Configure before connecting %s", async (adapterType) => {
+    settings.get.mockResolvedValue({ defaultEnvironmentId: "no-login" });
+    envApi.list.mockResolvedValue([
+      { id: "no-login", name: "No browser sign-in", status: "active", driver: "sandbox", config: { provider: "other" } },
+      { id: "login-env", name: "Browser sign-in", status: "active", driver: "sandbox", config: { provider: "daytona" } },
+    ]);
+    envApi.capabilities.mockResolvedValue({ sandboxProviders: { daytona: { supportsLoginPty: true } } });
+    api.getAdapterAuthSignal.mockResolvedValue({ status: "missing" });
+    await render(adapterType);
+    expect(container.textContent).toContain("does not support browser sign-in");
+    expect(container.querySelector('select[aria-label="Environment"]')).toBeNull();
+    await click("2Configure");
+    expect(container.querySelector('button[type="submit"]')).toBeNull();
+    expect([...container.querySelectorAll("button")].find(button => button.textContent?.trim() === "Run test")?.disabled).toBe(true);
+    await click("Connect model");
+    expect(api.testEnvironment).not.toHaveBeenCalled();
+    expect(api.hire).not.toHaveBeenCalled();
+    await click("2Configure");
+    const select = container.querySelector('select[aria-label="Environment"]') as HTMLSelectElement;
+    await act(async () => {
+      select.value = "login-env";
+      select.dispatchEvent(new Event("change", { bubbles: true }));
+    });
+    await settle();
+    expect(container.querySelector('select[aria-label="Environment"]')).toBeNull();
+    await click("Complete subscription login");
+    await click("Finish setup");
+    expect(api.hire.mock.calls[0][1]).toMatchObject({ defaultEnvironmentId: "login-env", runtimeConfig: { aiConnection: { method: "subscription" } } });
+  });
   it("waits for the resolved environment before allowing a credential-mode selection", async () => {
     let resolveEnvironments!: (value: object[]) => void;
     envApi.list.mockImplementationOnce(() => new Promise<object[]>(resolve => { resolveEnvironments = resolve; }));