Clean interrupted exe.dev fixtures and smoke every bundled CLI

Verify ownership before launcher cleanup when browser teardown is interrupted.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/exe-dev-live/stress.ts

@@ -35,6 +35,10 @@ try {
     leases.push(await hooks.onEnvironmentAcquireLease!({ ...scope, runId: randomUUID(), agentId: "agent-0" }));
     assert.equal(leases[0].metadata?.vmName, vmName);
   });
+  await check("every bundled legacy CLI starts without credentials", async () => {
+    const result = await execute(leases[0], "set -e; for cli in codex claude opencode grok gemini kimi hermes cursor-agent gh; do timeout 45 \"$cli\" --version; done", 300000);
+    assert.equal(result.exitCode, 0, result.stderr);
+  });
   const resourceBinding = leases[0].metadata?.environmentResourceBinding as NonNullable<Parameters<NonNullable<typeof hooks.onEnvironmentAcquireLease>>[0]["resourceBinding"]>;
   await check("eight independent leases share the bound VM", async () => {
     leases.push(...await Promise.all(Array.from({ length: 7 }, (_, i) => hooks.onEnvironmentAcquireLease!({ ...scope, resourceBinding, runId: randomUUID(), agentId: `agent-${i + 1}` }))));

tests/runner-e2e/exe-cleanup.test.ts

@@ -0,0 +1,28 @@
+import { createHash } from "node:crypto";
+import { beforeEach, expect, it, vi } from "vitest";
+const mocks = vi.hoisted(() => ({ ssh: vi.fn() }));
+vi.mock("../../packages/plugins/sandbox-providers/exe-dev/src/transport.js", () => ({ ssh: mocks.ssh, quote: (value: string) => `'${value}'` }));
+import { cleanupExeFixture } from "./exe-cleanup.js";
+const fixture = { vmName: "paperclip-e2e-0123456789abcdef0123", companyId: "company", environmentId: "environment", sshPrivateKey: "test-key" };
+beforeEach(() => mocks.ssh.mockReset());
+it("refuses an unrelated name before connecting", async () => {
+  await expect(cleanupExeFixture({ ...fixture, vmName: "personal-server" })).rejects.toThrow("non-campaign");
+  expect(mocks.ssh).not.toHaveBeenCalled();
+});
+it("refuses a matching name owned by another fixture", async () => {
+  mocks.ssh.mockResolvedValue(JSON.stringify({ vms: [{ vm_name: fixture.vmName, tags: ["other"] }] }));
+  await expect(cleanupExeFixture(fixture)).rejects.toThrow("ownership tag");
+  expect(mocks.ssh).toHaveBeenCalledTimes(1);
+});
+it("deletes only an owned VM and verifies its absence", async () => {
+  const tag = "paperclip-" + createHash("sha256").update("company\0environment").digest("hex").slice(0, 32);
+  mocks.ssh.mockResolvedValueOnce(JSON.stringify({ vms: [{ vm_name: fixture.vmName, tags: [tag] }] })).mockResolvedValueOnce("{}").mockResolvedValueOnce('{"vms":[]}');
+  await cleanupExeFixture(fixture);
+  expect(mocks.ssh.mock.calls[1][2]).toBe(`rm --json '${fixture.vmName}'`);
+  expect(mocks.ssh).toHaveBeenCalledTimes(3);
+});
+it("is idempotent when normal teardown already removed the VM", async () => {
+  mocks.ssh.mockResolvedValue('{"vms":[]}');
+  await cleanupExeFixture(fixture);
+  expect(mocks.ssh).toHaveBeenCalledTimes(1);
+});

tests/runner-e2e/exe-cleanup.ts

@@ -0,0 +1,21 @@
+import { createHash } from "node:crypto";
+import { ssh, quote } from "../../packages/plugins/sandbox-providers/exe-dev/src/transport.js";
+
+/** Destructive fixture cleanup requires the exact company/environment ownership tag. */
+export async function cleanupExeFixture(input: {
+  vmName: unknown; companyId: string; environmentId: string; sshPrivateKey?: string;
+}) {
+  if (typeof input.vmName !== "string" || !/^paperclip-e2e-[a-f0-9]{20}$/.test(input.vmName)) {
+    throw new Error("Refusing cleanup of a non-campaign VM");
+  }
+  if (!input.sshPrivateKey) throw new Error("Missing exe.dev cleanup credential");
+  const config = { sshPrivateKey: input.sshPrivateKey, strictHostKeyChecking: "accept-new" as const, timeoutMs: 120000 };
+  const listing = JSON.parse(await ssh(config, "exe.dev", "ls -l --json"));
+  const vm = listing.vms.find((entry: Record<string, unknown>) => entry.vm_name === input.vmName);
+  if (!vm) return;
+  const tag = "paperclip-" + createHash("sha256").update(`${input.companyId}\0${input.environmentId}`).digest("hex").slice(0, 32);
+  if (!vm.tags?.includes(tag)) throw new Error("Refusing deletion: VM ownership tag does not match this fixture");
+  await ssh(config, "exe.dev", `rm --json ${quote(input.vmName)}`);
+  const remaining = JSON.parse(await ssh(config, "exe.dev", "ls --json"));
+  if (remaining.vms.some((entry: Record<string, unknown>) => entry.vm_name === input.vmName)) throw new Error("exe.dev VM cleanup could not be confirmed");
+}

tests/runner-e2e/launch.ts

@@ -1,4 +1,5 @@
 import { randomBytes } from "node:crypto";
+import { cleanupExeFixture } from "./exe-cleanup.js";
 import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js";
 import { spawn } from "node:child_process";
 import { createWriteStream } from "node:fs";
@@ -880,6 +881,24 @@ async function runAttempt(input: {
       }
       const casePrivateDir = path.join(privateDir, "cases", candidate.task.id);
       const resultPath = path.join(casePrivateDir, "result.json");
+      if (candidate.environment.id === "exe-dev") {
+        try {
+          // Playwright may have been interrupted before fixture teardown. The
+          // launcher owns this private snapshot and can still reclaim the VM.
+          const snapshot = await readFile(path.join(casePrivateDir, "snapshots", "fixtures.json"), "utf8")
+            .then((contents) => JSON.parse(contents))
+            .catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; });
+          if (snapshot) {
+            if (snapshot.executionId !== candidate.id) throw new Error("Fixture cleanup execution mismatch");
+            await cleanupExeFixture({ vmName: snapshot.persistedEnvironment?.config?.vmName,
+              companyId: snapshot.companyId, environmentId: snapshot.environmentId,
+              sshPrivateKey: process.env.EXE_DEV_SSH_PRIVATE_KEY });
+          }
+        } catch (error) {
+          result = { ...result, status: "failed", failureClass: "cleanup_failure", cleanup: "failed",
+            error: `exe.dev fixture cleanup failed: ${String(error)}` };
+        }
+      }
       const uploadDir = path.join(
         resultsRoot,
         campaignId,

tests/runner-e2e/live-fixtures.ts

@@ -1,5 +1,4 @@
-import { createHash } from "node:crypto";
-import { ssh, quote } from "../../packages/plugins/sandbox-providers/exe-dev/src/transport.js";
+import { cleanupExeFixture } from "./exe-cleanup.js";
 import path from "node:path";
 import { FixtureRegistry } from "./fixture-registry.js";
 import type { RunnerApi } from "./api.js";
@@ -213,20 +212,13 @@ export async function setupLiveFixtures(input: {
     },
     async teardown(environment, resolved) {
       if (execution.environment.driver === "sandbox") {
-        await deleteDaytonaEnvironment(api, environment.id);
-        if (execution.environment.id === "exe-dev") {
-          const vmName = environment.config?.vmName;
-          if (typeof vmName !== "string" || !/^paperclip-e2e-[a-f0-9]{20}$/.test(vmName)) throw new Error("Refusing cleanup of a non-campaign VM");
-          const sshConfig = { sshPrivateKey: input.credentials.EXE_DEV_SSH_PRIVATE_KEY, strictHostKeyChecking: "accept-new" as const, timeoutMs: 120000 };
-          const listing = JSON.parse(await ssh(sshConfig, "exe.dev", "ls -l --json"));
-          const vm = listing.vms.find((entry: Record<string, unknown>) => entry.vm_name === vmName);
-          if (vm) {
-            const company = value<CompanyRecord>(resolved, "company");
-            const tag = "paperclip-" + createHash("sha256").update(`${company.id}\0${environment.id}`).digest("hex").slice(0, 32);
-            if (!vm.tags?.includes(tag)) throw new Error("Refusing deletion: VM ownership tag does not match this fixture");
-            await ssh(sshConfig, "exe.dev", `rm --json ${quote(vmName)}`);
-            const remaining = JSON.parse(await ssh(sshConfig, "exe.dev", "ls --json"));
-            if (remaining.vms.some((entry: Record<string, unknown>) => entry.vm_name === vmName)) throw new Error("exe.dev VM cleanup could not be confirmed");
+        try {
+          await deleteDaytonaEnvironment(api, environment.id);
+        } finally {
+          if (execution.environment.id === "exe-dev") {
+            await cleanupExeFixture({ vmName: environment.config?.vmName,
+              companyId: value<CompanyRecord>(resolved, "company").id,
+              environmentId: environment.id, sshPrivateKey: input.credentials.EXE_DEV_SSH_PRIVATE_KEY });
           }
         }
       }