diff --git a/tests/exe-dev-live/stress.ts b/tests/exe-dev-live/stress.ts index 7deeb3694e7..fbd67b6df47 100644 --- a/tests/exe-dev-live/stress.ts +++ b/tests/exe-dev-live/stress.ts @@ -35,6 +35,10 @@ try { leases.push(await hooks.onEnvironmentAcquireLease!({ ...scope, runId: randomUUID(), agentId: "agent-0" })); assert.equal(leases[0].metadata?.vmName, vmName); }); + await check("every bundled legacy CLI starts without credentials", async () => { + const result = await execute(leases[0], "set -e; for cli in codex claude opencode grok gemini kimi hermes cursor-agent gh; do timeout 45 \"$cli\" --version; done", 300000); + assert.equal(result.exitCode, 0, result.stderr); + }); const resourceBinding = leases[0].metadata?.environmentResourceBinding as NonNullable>[0]["resourceBinding"]>; await check("eight independent leases share the bound VM", async () => { leases.push(...await Promise.all(Array.from({ length: 7 }, (_, i) => hooks.onEnvironmentAcquireLease!({ ...scope, resourceBinding, runId: randomUUID(), agentId: `agent-${i + 1}` })))); diff --git a/tests/runner-e2e/exe-cleanup.test.ts b/tests/runner-e2e/exe-cleanup.test.ts new file mode 100644 index 00000000000..e83fcbaae95 --- /dev/null +++ b/tests/runner-e2e/exe-cleanup.test.ts @@ -0,0 +1,28 @@ +import { createHash } from "node:crypto"; +import { beforeEach, expect, it, vi } from "vitest"; +const mocks = vi.hoisted(() => ({ ssh: vi.fn() })); +vi.mock("../../packages/plugins/sandbox-providers/exe-dev/src/transport.js", () => ({ ssh: mocks.ssh, quote: (value: string) => `'${value}'` })); +import { cleanupExeFixture } from "./exe-cleanup.js"; +const fixture = { vmName: "paperclip-e2e-0123456789abcdef0123", companyId: "company", environmentId: "environment", sshPrivateKey: "test-key" }; +beforeEach(() => mocks.ssh.mockReset()); +it("refuses an unrelated name before connecting", async () => { + await expect(cleanupExeFixture({ ...fixture, vmName: "personal-server" })).rejects.toThrow("non-campaign"); + expect(mocks.ssh).not.toHaveBeenCalled(); +}); +it("refuses a matching name owned by another fixture", async () => { + mocks.ssh.mockResolvedValue(JSON.stringify({ vms: [{ vm_name: fixture.vmName, tags: ["other"] }] })); + await expect(cleanupExeFixture(fixture)).rejects.toThrow("ownership tag"); + expect(mocks.ssh).toHaveBeenCalledTimes(1); +}); +it("deletes only an owned VM and verifies its absence", async () => { + const tag = "paperclip-" + createHash("sha256").update("company\0environment").digest("hex").slice(0, 32); + mocks.ssh.mockResolvedValueOnce(JSON.stringify({ vms: [{ vm_name: fixture.vmName, tags: [tag] }] })).mockResolvedValueOnce("{}").mockResolvedValueOnce('{"vms":[]}'); + await cleanupExeFixture(fixture); + expect(mocks.ssh.mock.calls[1][2]).toBe(`rm --json '${fixture.vmName}'`); + expect(mocks.ssh).toHaveBeenCalledTimes(3); +}); +it("is idempotent when normal teardown already removed the VM", async () => { + mocks.ssh.mockResolvedValue('{"vms":[]}'); + await cleanupExeFixture(fixture); + expect(mocks.ssh).toHaveBeenCalledTimes(1); +}); diff --git a/tests/runner-e2e/exe-cleanup.ts b/tests/runner-e2e/exe-cleanup.ts new file mode 100644 index 00000000000..910c79f840e --- /dev/null +++ b/tests/runner-e2e/exe-cleanup.ts @@ -0,0 +1,21 @@ +import { createHash } from "node:crypto"; +import { ssh, quote } from "../../packages/plugins/sandbox-providers/exe-dev/src/transport.js"; + +/** Destructive fixture cleanup requires the exact company/environment ownership tag. */ +export async function cleanupExeFixture(input: { + vmName: unknown; companyId: string; environmentId: string; sshPrivateKey?: string; +}) { + if (typeof input.vmName !== "string" || !/^paperclip-e2e-[a-f0-9]{20}$/.test(input.vmName)) { + throw new Error("Refusing cleanup of a non-campaign VM"); + } + if (!input.sshPrivateKey) throw new Error("Missing exe.dev cleanup credential"); + const config = { sshPrivateKey: input.sshPrivateKey, strictHostKeyChecking: "accept-new" as const, timeoutMs: 120000 }; + const listing = JSON.parse(await ssh(config, "exe.dev", "ls -l --json")); + const vm = listing.vms.find((entry: Record) => entry.vm_name === input.vmName); + if (!vm) return; + const tag = "paperclip-" + createHash("sha256").update(`${input.companyId}\0${input.environmentId}`).digest("hex").slice(0, 32); + if (!vm.tags?.includes(tag)) throw new Error("Refusing deletion: VM ownership tag does not match this fixture"); + await ssh(config, "exe.dev", `rm --json ${quote(input.vmName)}`); + const remaining = JSON.parse(await ssh(config, "exe.dev", "ls --json")); + if (remaining.vms.some((entry: Record) => entry.vm_name === input.vmName)) throw new Error("exe.dev VM cleanup could not be confirmed"); +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 5fb923aac9d..0d343ac0972 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,4 +1,5 @@ import { randomBytes } from "node:crypto"; +import { cleanupExeFixture } from "./exe-cleanup.js"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; @@ -880,6 +881,24 @@ async function runAttempt(input: { } const casePrivateDir = path.join(privateDir, "cases", candidate.task.id); const resultPath = path.join(casePrivateDir, "result.json"); + if (candidate.environment.id === "exe-dev") { + try { + // Playwright may have been interrupted before fixture teardown. The + // launcher owns this private snapshot and can still reclaim the VM. + const snapshot = await readFile(path.join(casePrivateDir, "snapshots", "fixtures.json"), "utf8") + .then((contents) => JSON.parse(contents)) + .catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (snapshot) { + if (snapshot.executionId !== candidate.id) throw new Error("Fixture cleanup execution mismatch"); + await cleanupExeFixture({ vmName: snapshot.persistedEnvironment?.config?.vmName, + companyId: snapshot.companyId, environmentId: snapshot.environmentId, + sshPrivateKey: process.env.EXE_DEV_SSH_PRIVATE_KEY }); + } + } catch (error) { + result = { ...result, status: "failed", failureClass: "cleanup_failure", cleanup: "failed", + error: `exe.dev fixture cleanup failed: ${String(error)}` }; + } + } const uploadDir = path.join( resultsRoot, campaignId, diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 8438adb24e5..7622ae2d388 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,5 +1,4 @@ -import { createHash } from "node:crypto"; -import { ssh, quote } from "../../packages/plugins/sandbox-providers/exe-dev/src/transport.js"; +import { cleanupExeFixture } from "./exe-cleanup.js"; import path from "node:path"; import { FixtureRegistry } from "./fixture-registry.js"; import type { RunnerApi } from "./api.js"; @@ -213,20 +212,13 @@ export async function setupLiveFixtures(input: { }, async teardown(environment, resolved) { if (execution.environment.driver === "sandbox") { - await deleteDaytonaEnvironment(api, environment.id); - if (execution.environment.id === "exe-dev") { - const vmName = environment.config?.vmName; - if (typeof vmName !== "string" || !/^paperclip-e2e-[a-f0-9]{20}$/.test(vmName)) throw new Error("Refusing cleanup of a non-campaign VM"); - const sshConfig = { sshPrivateKey: input.credentials.EXE_DEV_SSH_PRIVATE_KEY, strictHostKeyChecking: "accept-new" as const, timeoutMs: 120000 }; - const listing = JSON.parse(await ssh(sshConfig, "exe.dev", "ls -l --json")); - const vm = listing.vms.find((entry: Record) => entry.vm_name === vmName); - if (vm) { - const company = value(resolved, "company"); - const tag = "paperclip-" + createHash("sha256").update(`${company.id}\0${environment.id}`).digest("hex").slice(0, 32); - if (!vm.tags?.includes(tag)) throw new Error("Refusing deletion: VM ownership tag does not match this fixture"); - await ssh(sshConfig, "exe.dev", `rm --json ${quote(vmName)}`); - const remaining = JSON.parse(await ssh(sshConfig, "exe.dev", "ls --json")); - if (remaining.vms.some((entry: Record) => entry.vm_name === vmName)) throw new Error("exe.dev VM cleanup could not be confirmed"); + try { + await deleteDaytonaEnvironment(api, environment.id); + } finally { + if (execution.environment.id === "exe-dev") { + await cleanupExeFixture({ vmName: environment.config?.vmName, + companyId: value(resolved, "company").id, + environmentId: environment.id, sshPrivateKey: input.credentials.EXE_DEV_SSH_PRIVATE_KEY }); } } }