From 056703424a20279870eadfa46a4979fb5eefafcf Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 17 Sep 2026 09:14:54 -0500 Subject: [PATCH] Add experimental durable exe.dev agent environments Keep one VM per environment with independent lease cgroups, persistent workspaces, immutable runner images, and an explicit experimental gate. Add live compatibility, recovery, browser-preview, and stress campaigns. Co-Authored-By: Paperclip --- .github/workflows/docker-cloud.yml | 2 +- .github/workflows/exe-dev-image.yml | 35 + .github/workflows/release.yml | 2 +- Dockerfile | 2 +- doc/plans/2026-09-17-exe-dev-environments.md | 50 + docker/exe-dev-runner/Dockerfile | 150 +++ docker/exe-dev-runner/Dockerfile.dockerignore | 51 + docker/exe-dev-runner/lease.mjs | 89 ++ .../adapter-utils/src/execution-target.ts | 28 +- .../sandbox-providers/exe-dev/README.md | 162 ++- .../sandbox-providers/exe-dev/package.json | 2 +- .../sandbox-providers/exe-dev/src/manifest.ts | 210 +-- .../exe-dev/src/plugin.test.ts | 971 ++------------ .../sandbox-providers/exe-dev/src/plugin.ts | 1191 ++++------------- .../sandbox-providers/exe-dev/src/ssh-key.ts | 60 + .../exe-dev/src/transport.ts | 70 + packages/plugins/sdk/src/index.ts | 1 + packages/plugins/sdk/src/protocol.ts | 9 + packages/shared/src/feature-catalog.ts | 7 + packages/shared/src/types/instance.ts | 2 + packages/shared/src/validators/instance.ts | 1 + scripts/build-exe-dev-image.ts | 30 + scripts/check-node-version-policy.mjs | 2 +- .../exe-environment-resource.test.ts | 61 + server/src/__tests__/redaction.test.ts | 11 + server/src/redaction.ts | 10 +- server/src/routes/environments.ts | 11 +- server/src/services/environment-probe.ts | 4 +- .../services/environment-resource-binding.ts | 34 + server/src/services/environment-runtime.ts | 13 + server/src/services/environments.ts | 7 +- server/src/services/exe-environment-gate.ts | 19 + server/src/services/instance-settings.ts | 2 + tests/exe-dev-live/preview.ts | 50 + tests/exe-dev-live/stress.ts | 172 +++ tests/exe-dev-live/tsconfig.json | 5 + tests/runner-e2e/billing.test.ts | 5 + tests/runner-e2e/billing.ts | 9 + tests/runner-e2e/catalog.test.ts | 8 +- tests/runner-e2e/catalog.ts | 55 +- tests/runner-e2e/chat-cases.ts | 2 +- tests/runner-e2e/everyday-cases.ts | 2 +- tests/runner-e2e/exe-shared-agents.ts | 55 + tests/runner-e2e/first-task-cases.ts | 2 +- tests/runner-e2e/harness-env.ts | 2 +- tests/runner-e2e/launch.ts | 3 + tests/runner-e2e/live-fixtures.ts | 33 +- tests/runner-e2e/report-catalog.ts | 4 +- tests/runner-e2e/result-validation.ts | 4 +- tests/runner-e2e/runner-wss-tunnel.test.ts | 11 + tests/runner-e2e/runner-wss-tunnel.ts | 52 + tests/runner-e2e/runner.spec.ts | 29 +- tests/runner-e2e/server.ts | 9 + tests/runner-e2e/types.ts | 13 +- .../InstanceExperimentalSettings.test.tsx | 8 + ui/src/pages/InstanceExperimentalSettings.tsx | 12 + 56 files changed, 1782 insertions(+), 2062 deletions(-) create mode 100644 .github/workflows/exe-dev-image.yml create mode 100644 doc/plans/2026-09-17-exe-dev-environments.md create mode 100644 docker/exe-dev-runner/Dockerfile create mode 100644 docker/exe-dev-runner/Dockerfile.dockerignore create mode 100644 docker/exe-dev-runner/lease.mjs create mode 100644 packages/plugins/sandbox-providers/exe-dev/src/ssh-key.ts create mode 100644 packages/plugins/sandbox-providers/exe-dev/src/transport.ts create mode 100644 scripts/build-exe-dev-image.ts create mode 100644 server/src/__tests__/exe-environment-resource.test.ts create mode 100644 server/src/services/environment-resource-binding.ts create mode 100644 server/src/services/exe-environment-gate.ts create mode 100644 tests/exe-dev-live/preview.ts create mode 100644 tests/exe-dev-live/stress.ts create mode 100644 tests/exe-dev-live/tsconfig.json create mode 100644 tests/runner-e2e/exe-shared-agents.ts create mode 100644 tests/runner-e2e/runner-wss-tunnel.test.ts create mode 100644 tests/runner-e2e/runner-wss-tunnel.ts diff --git a/.github/workflows/docker-cloud.yml b/.github/workflows/docker-cloud.yml index 81e608c9277..eca4f4e23e6 100644 --- a/.github/workflows/docker-cloud.yml +++ b/.github/workflows/docker-cloud.yml @@ -201,7 +201,7 @@ jobs: build-args: | USER_UID=1001 USER_GID=1001 - CLOUD_BUNDLED_PLUGINS=daytona + CLOUD_BUNDLED_PLUGINS=daytona exe-dev CLOUD_BUNDLED_SERVER_DEPS=@sentry/node PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }} PAPERCLIP_BUILD_COMMIT=${{ github.sha }} diff --git a/.github/workflows/exe-dev-image.yml b/.github/workflows/exe-dev-image.yml new file mode 100644 index 00000000000..a983e6cf6e0 --- /dev/null +++ b/.github/workflows/exe-dev-image.yml @@ -0,0 +1,35 @@ +name: Experimental exe.dev image +on: + workflow_dispatch: +permissions: + contents: read + packages: write +concurrency: + group: exe-dev-image-${{ github.ref }} + cancel-in-progress: false +jobs: + image: + if: github.repository == 'paperclipai/paperclip' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: '24' + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: pnpm install --frozen-lockfile + - name: Build and publish immutable experimental image + run: node cli/node_modules/tsx/dist/cli.mjs scripts/build-exe-dev-image.ts --push --metadata-file /tmp/exe-image.json + - uses: actions/upload-artifact@v4 + with: + name: exe-dev-image-digest + path: /tmp/exe-image.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 878899cb248..8f1e9703839 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -235,7 +235,7 @@ jobs: outputs: type=docker,dest=${{ runner.temp }}/preview-image.tar tags: ghcr.io/paperclipai/paperclip:sha-${{ inputs.source_ref }}-cloud build-args: | - CLOUD_BUNDLED_PLUGINS=daytona + CLOUD_BUNDLED_PLUGINS=daytona exe-dev CLOUD_BUNDLED_SERVER_DEPS=@sentry/node PAPERCLIP_BUILD_COMMIT=${{ inputs.source_ref }} PAPERCLIP_BUILD_VERSION=0.0.0-preview.g${{ inputs.source_ref }} diff --git a/Dockerfile b/Dockerfile index a3d046c96ea..5db11cbbf33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -226,7 +226,7 @@ CMD ["node", "--import", "./server/node_modules/tsx/dist/loader.mjs", "server/di # actually auto-install belongs here — every entry adds its node_modules # to the image. Growing the list is a one-line workflow change. FROM build AS cloud-plugins -ARG CLOUD_BUNDLED_PLUGINS="daytona" +ARG CLOUD_BUNDLED_PLUGINS="daytona exe-dev" RUN set -eu; \ for name in $CLOUD_BUNDLED_PLUGINS; do \ dir="packages/plugins/sandbox-providers/$name"; \ diff --git a/doc/plans/2026-09-17-exe-dev-environments.md b/doc/plans/2026-09-17-exe-dev-environments.md new file mode 100644 index 00000000000..3cf5ba7cd75 --- /dev/null +++ b/doc/plans/2026-09-17-exe-dev-environments.md @@ -0,0 +1,50 @@ +# Experimental durable exe.dev environments + +## Product contract + +One environment binds one durable VM for a trusted group of agents. Separate +leases, homes, and worktrees isolate ordinary concurrent work; the VM is the trust +boundary. Bring an exe.dev account and an SSH key. Create from a pinned compatible +image or attach a compatible existing VM. Keep Daytona as Cloud's default. + +The experiment is off by default in both products. New environment creation, +probing, discovery, and execution require its flag. Cleanup remains available +when disabled. VM deletion and replacement are always explicit. A VM resource +identity must outlive individual run leases and survive controller restart. + +Use existing workspace copyback and native checkpoint semantics. The durable disk +is authoritative between runs. Cloud backup coverage is its existing backed-up +volume for copied/checkpointed paths; no new whole-disk or continuous S3 backup. +No service registry, port allocator, or database-backup feature is in scope. + +## Acceptance campaign + +1. Offline contracts: flag defaults and server denials, secret references, host + key checks, immutable image requirement, cross-company/identity rejection, + atomic resource binding, idempotent cleanup, and no VM deletion in hooks. +2. Reproducible image: frozen dependency lock; runner/provider pack from one source + revision; all legacy CLI commands available; no package installation at boot; + boot a fresh VM from the exact published digest. +3. Browser matrix: legacy Codex, Claude, OpenCode and native Codex, OpenCode, + ACPX Claude, ACPX Codex, each across message completion, plan revision/approval, + and question/resume. Verify visible status, output, runtime mode, transport, + usage, artifacts, and cleanup. Missing credentials are failures, not skips. +4. Shared VM stress: concurrent native/native and native/legacy agents, independent + homes/worktrees, competing first acquisitions, cancellation of one agent while + others progress, command timeout, large binary and UTF-8 transfers, noisy logs, + and exact exit receipts. +5. Persistence: untracked/dirty/multiple Git repositories, remote edits between + runs, concurrent host edits and conflict preservation, controller/plugin + restart, VM reboot, SSH/WSS loss, missing VM, and explicit replacement from + verified copyback/checkpoints. Never overwrite an unverified surviving disk. +6. Preview: Vite/HMR through a private exe.dev URL, service alive after agent exit + and cancellation, unauthenticated visitor denied, authenticated browser allowed, + 60-minute idle and overnight soak when scheduled. Record actual sleep/wake + behavior rather than infer it from marketing language. +7. Cloud: release-scoped opt-in installs the plugin, canonical-host outbound WSS, + existing Daytona provisioning unaffected, managed-config and runtime-envelope + tests, tenant-volume backup coverage documented. + +Live logs, screenshots, image digests, and campaign results are generated evidence +and stay outside source control. A passing provider stress script alone does not +establish browser acceptance or qualify every runner profile. diff --git a/docker/exe-dev-runner/Dockerfile b/docker/exe-dev-runner/Dockerfile new file mode 100644 index 00000000000..94eacff762a --- /dev/null +++ b/docker/exe-dev-runner/Dockerfile @@ -0,0 +1,150 @@ +# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e + +# Keep the Rust toolchain explicit so the runner artifact is reproducible and +# is always built for the same platform as the final Daytona image. Base-image +# digests are the reviewed linux/amd64 manifests; the tags are annotations. +FROM rust:1.97-bookworm@sha256:408fe88047cef61a2087653b0c5255fa51c0f2d6d94ddedd7a2562a9b91a46f6 AS runnerd-build + +WORKDIR /workspace/packages/paperclip-runner/runner +COPY packages/paperclip-runner/runner/Cargo.toml packages/paperclip-runner/runner/Cargo.lock ./ +COPY packages/paperclip-runner/runner/crates ./crates +COPY packages/paperclip-runner/protocol ../protocol +RUN cargo build --locked --release -p paperclip-runner-core --bin paperclip-runnerd \ + && strip /workspace/packages/paperclip-runner/runner/target/release/paperclip-runnerd + +FROM node:24-bookworm@sha256:9137a20e25879e0b557227b57e3ee4e9af4bde29eb3db66134cd1723e84f830b AS provider-pack-build +RUN corepack enable && corepack prepare pnpm@9.15.4 --activate +WORKDIR /workspace +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc tsconfig.base.json ./ +COPY patches ./patches +COPY scripts/link-plugin-dev-sdk.mjs ./scripts/link-plugin-dev-sdk.mjs +# Build from the committed workspace lock; never resolve new dependency versions +# while creating an image. Include the workspace graph for source-owned patches. +COPY packages ./packages +COPY server/package.json ./server/package.json +COPY ui/package.json ./ui/package.json +COPY cli/package.json ./cli/package.json +# The complete resolved lock (including transitive integrity hashes) is reviewed. +# Reject registry-time drift BEFORE installing packages or running lifecycle code. +# Refresh this digest together with source/provider dependency changes. +ARG PAPERCLIP_RUNNER_LOCK_SHA256=af3b879fea127a608b0f9279627f357638299441bbf41835f0b91ab0162e441c +RUN printf '%s pnpm-lock.yaml\n' "${PAPERCLIP_RUNNER_LOCK_SHA256}" > /tmp/provider-lock.sha256 \ + && sha256sum -c /tmp/provider-lock.sha256 \ + && pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...' +ARG PAPERCLIP_RUNNER_SOURCE_REVISION +RUN test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" +RUN pnpm --filter @paperclipai/paperclip-runner build:typescript \ + && PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_RUNNER_SOURCE_REVISION}" \ + node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack \ + && chmod -R a+rX /provider-pack + +# Fleet sandbox base image. Keep this section aligned with +# paperclipai/paperclip-cloud/fleet-sandbox-image/Dockerfile. The only Paperclip +# runner-specific addition is /usr/local/bin/paperclip-runnerd below. +FROM ghcr.io/boldsoftware/exeuntu@sha256:5db55eada806b5c4c72eda61f11643a62d34ada2617db91152e4186a47cd730e + +USER root +COPY --from=provider-pack-build /usr/local/ /usr/local/ +RUN apt-get update && apt-get install -y --no-install-recommends python3-venv rsync && rm -rf /var/lib/apt/lists/* + +ENV PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=/opt/paperclip-runner/provider-pack +ENV PATH=${PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT}/node_modules/.bin:/usr/local/share/nvm/current/bin:/usr/local/python/current/bin:/usr/local/py-utils/bin:${PATH} + +# Share the pack's latest stable Codex, Claude and OpenCode with every adapter. +# Never add a second CLI version to work around a runner compatibility pin. +RUN npm uninstall -g @anthropic-ai/claude-code @openai/codex opencode-ai \ + && npm install -g \ + @xai-official/grok@1.0.13 \ + @google/gemini-cli@0.58.0 \ + @moonshot-ai/kimi-code@0.41.0 \ + && npm cache clean --force + +# Hermes requires Python >=3.11,<3.14; the image's default Python is newer. +# Use the distro interpreter in a dedicated environment without changing PATH +# or keeping an older Hermes CLI. Everything is installed before sandbox boot. +# Debian 13 in the sandbox base already includes Python 3.13 and python3-venv. +RUN /usr/bin/python3 -m venv /opt/hermes \ + && /opt/hermes/bin/pip install --no-cache-dir hermes-agent==0.19.0 \ + && ln -sf /opt/hermes/bin/hermes /usr/local/bin/hermes + +ARG CURSOR_VERSION=2026.09.02-c22c1a3 +ARG CURSOR_SHA256_AMD64=b73b59854762535c0fc20d7ccc51c3b5a356a851491088d60a362be48750f53c +RUN set -eu; \ + arch="$(dpkg --print-architecture)"; \ + [ "$arch" = "amd64" ] || { echo "FATAL: cursor pin only covers amd64, not $arch" >&2; exit 1; }; \ + dir="/opt/cursor/versions/${CURSOR_VERSION}"; mkdir -p "$dir"; \ + curl -fsSL "https://downloads.cursor.com/lab/${CURSOR_VERSION}/linux/x64/agent-cli-package.tar.gz" -o /tmp/cursor.tgz; \ + printf '%s /tmp/cursor.tgz\n' "${CURSOR_SHA256_AMD64}" > /tmp/cursor.sha256; \ + sha256sum -c /tmp/cursor.sha256; \ + tar --strip-components=1 -xzf /tmp/cursor.tgz -C "$dir"; \ + rm -f /tmp/cursor.tgz /tmp/cursor.sha256; \ + ln -sf "$dir/cursor-agent" /usr/local/bin/cursor-agent; \ + ln -sf /usr/local/bin/cursor-agent /usr/local/bin/agent; \ + chmod -R a+rX /opt/cursor + +ARG GH_VERSION=2.100.0 +ARG GH_SHA256_AMD64=e4d4bb4498e8d007abe545b6568926793ace1b6447da598294a610018cb164be +RUN set -eu; \ + arch="$(dpkg --print-architecture)"; \ + [ "$arch" = "amd64" ] || { echo "FATAL: gh pin only covers amd64, not $arch" >&2; exit 1; }; \ + curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" -o /tmp/gh.tgz; \ + printf '%s /tmp/gh.tgz\n' "${GH_SHA256_AMD64}" > /tmp/gh.sha256; \ + sha256sum -c /tmp/gh.sha256; \ + tar -xzf /tmp/gh.tgz -C /tmp; \ + install "/tmp/gh_${GH_VERSION}_linux_amd64/bin/gh" /usr/local/bin/gh; \ + rm -rf /tmp/gh.tgz /tmp/gh.sha256 "/tmp/gh_${GH_VERSION}_linux_amd64" + +COPY --from=runnerd-build /workspace/packages/paperclip-runner/runner/target/release/paperclip-runnerd /usr/local/bin/paperclip-runnerd +COPY --from=provider-pack-build /provider-pack /opt/paperclip-runner/provider-pack +RUN set -eu; for cli in codex claude opencode; do \ + printf '#!/bin/sh\nexec /opt/paperclip-runner/provider-pack/node_modules/.bin/%s "$@"\n' "$cli" > "/usr/local/bin/$cli"; \ + chmod 755 "/usr/local/bin/$cli"; \ + done + +# Keep revision-dependent metadata below the stable agent CLI installation +# layers. A source-only image miss can then reuse those expensive layers from +# the trusted registry cache. +ARG PAPERCLIP_RUNNER_CONTENT_ID +ARG PAPERCLIP_RUNNER_SOURCE_REVISION +RUN test -n "${PAPERCLIP_RUNNER_CONTENT_ID}" \ + && test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" + +RUN set -eu; \ + printf '%s\n' 'export PATH=/opt/paperclip-runner/provider-pack/node_modules/.bin:$PATH' \ + > /etc/profile.d/01-paperclip-runner-provider-pack.sh; \ + chmod 0644 /etc/profile.d/01-paperclip-runner-provider-pack.sh; \ + for command_name in acpx claude-agent-acp codex-acp claude codex grok gemini kimi opencode cursor-agent agent hermes gh paperclip-runnerd; do \ + command -v "$command_name" >/dev/null || { echo "FATAL: $command_name not on PATH after build" >&2; exit 1; }; \ + done; \ + metadata="$(paperclip-runnerd --build-metadata)"; \ + /opt/paperclip-runner/provider-pack/node_modules/node/bin/node -e "const [major,minor,patch]=process.versions.node.split('.').map(Number); if (major<24 || (major===24 && (minor<11 || (minor===11 && patch<0)))) process.exit(1)"; \ + test -f /opt/paperclip-runner/provider-pack/provider-pack.json; \ + printf '%s' "$metadata" | grep -q '"dial_ws_loopback"'; \ + printf '%s' "$metadata" | grep -q '"dial_wss"'; \ + printf '%s' "$metadata" | grep -q '"listen_ws"'; \ + echo "all agent CLIs and runnerd transport modes are available" + +USER exedev + +RUN /bin/sh -lc 'set -eu; \ + test -r /opt/paperclip-runner/provider-pack/provider-pack.json; \ + /opt/paperclip-runner/provider-pack/node_modules/node/bin/node -e \ + "JSON.parse(require(\"node:fs\").readFileSync(\"/opt/paperclip-runner/provider-pack/provider-pack.json\", \"utf8\"))"; \ + for command_name in acpx claude-agent-acp codex-acp; do \ + command -v "$command_name" >/dev/null || { echo "FATAL: $command_name not on PATH for exedev user" >&2; exit 1; }; \ + done; \ + test "$(acpx --version)" = "0.13.1"; \ + test "$(claude-agent-acp --version)" = "0.73.0"; \ + test "$(codex-acp --version)" = "@agentclientprotocol/codex-acp 1.6.2"' + +LABEL io.paperclip.runner.content-id="${PAPERCLIP_RUNNER_CONTENT_ID}" \ + org.opencontainers.image.revision="${PAPERCLIP_RUNNER_SOURCE_REVISION}" + +USER root +COPY docker/exe-dev-runner/lease.mjs /opt/paperclip-exe/lease.mjs +RUN printf '{"schema":1,"sourceRevision":"%s","contentId":"%s"}\n' "$PAPERCLIP_RUNNER_SOURCE_REVISION" "$PAPERCLIP_RUNNER_CONTENT_ID" > /opt/paperclip-exe/image.json \ + && chmod -R a+rX /opt/paperclip-exe +LABEL exe.dev/login-user="exedev" exe.dev/install-shelley="false" io.paperclip.exe.schema="1" +CMD ["/sbin/init"] +LABEL org.opencontainers.image.source="https://github.com/paperclipai/paperclip" \ + org.opencontainers.image.description="Experimental Paperclip durable exe.dev runner image" diff --git a/docker/exe-dev-runner/Dockerfile.dockerignore b/docker/exe-dev-runner/Dockerfile.dockerignore new file mode 100644 index 00000000000..da12c9914f9 --- /dev/null +++ b/docker/exe-dev-runner/Dockerfile.dockerignore @@ -0,0 +1,51 @@ +.git +.github +.paperclip +.pnpm-store +node_modules +**/node_modules +coverage +data +tmp +*.log +packages/paperclip-runner/dist +packages/paperclip-runner/runner/target +packages/paperclip-runner/devtools +packages/paperclip-runner/docs +packages/paperclip-runner/examples +packages/paperclip-runner/test +packages/paperclip-runner/test-fixtures +packages/paperclip-runner/test-support +packages/paperclip-runner/**/*.md +packages/paperclip-runner/**/*.spec.ts +packages/paperclip-runner/**/*.spec.tsx +packages/paperclip-runner/**/*.test.cjs +packages/paperclip-runner/**/*.test.cts +packages/paperclip-runner/**/*.test.js +packages/paperclip-runner/**/*.test.jsx +packages/paperclip-runner/**/*.test.mjs +packages/paperclip-runner/**/*.test.mts +packages/paperclip-runner/**/*.test.ts +packages/paperclip-runner/**/*.test.tsx +packages/paperclip-runner/runner/crates/*/tests +packages/paperclip-runner/scripts/*-smoke.mjs +# Exceptions (last match wins): the image build re-runs the runner's +# generated-file drift checks, so their committed outputs and inputs must +# survive the slimming above. 2026-09-04: the *.md rule stripped the +# committed capability contract out of the context and every image build +# on master failed its drift check — .github/docker-context-checks.Dockerfile +# now guards this in PR CI. +!packages/paperclip-runner/generated/** +!packages/paperclip-runner/docs/capability-contract.md +# check:runner-workflow-traceability access()es every regression test the +# stress-traceability spec names — those are src/**/*.test.ts files, so +# they must survive the *.test.ts rule above. +!packages/paperclip-runner/src/**/*.test.ts +!packages/paperclip-runner/src/**/*.test.tsx + +# Only source-owned build inputs; never local evidence, outputs, or credentials. +**/dist +**/dist-* +tests/runner-e2e/results +.env* +**/.env* diff --git a/docker/exe-dev-runner/lease.mjs b/docker/exe-dev-runner/lease.mjs new file mode 100644 index 00000000000..70ad852ebed --- /dev/null +++ b/docker/exe-dev-runner/lease.mjs @@ -0,0 +1,89 @@ +// Image-owned execution supervisor. One systemd cgroup per lease, independent +// sockets per command. Closing SSH is transport loss, never proof of termination. +import net from 'node:net'; +import fs from 'node:fs'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +const [mode, root] = process.argv.slice(2); +if (!root || !/^\/var\/lib\/paperclip-exe\/[a-f0-9]{32}\/[a-f0-9]{32}$/.test(root)) { + throw new Error('Invalid lease root'); +} +const socketPath = path.join(root, 'control.sock'); +const MAX_FRAME = 16 * 1024 * 1024; +const send = (socket, value) => !socket.destroyed && socket.write(JSON.stringify(value) + '\n'); + +function frames(stream, onFrame) { + let pending = ''; + stream.setEncoding('utf8'); + stream.on('data', (data) => { + pending += data; + if (pending.length > MAX_FRAME) return stream.destroy(new Error('Frame too large')); + for (;;) { + const newline = pending.indexOf('\n'); + if (newline < 0) break; + const line = pending.slice(0, newline); + pending = pending.slice(newline + 1); + try { onFrame(JSON.parse(line)); } + catch { stream.destroy(new Error('Invalid frame')); } + } + }); +} + +if (mode === 'serve') { + process.umask(0o077); + fs.rmSync(socketPath, { force: true }); + const server = net.createServer((socket) => { + let child; + let timer; + let done = false; + let timedOut = false; + const terminate = () => { + if (!child?.pid || done) return; + try { process.kill(-child.pid, 'SIGKILL'); } catch {} + }; + socket.on('error', () => {}); + // Commands survive a disconnected SSH client. The lease's cgroup remains + // authoritative; cancellation and expiry explicitly stop it from outside. + frames(socket, (message) => { + if (message.type === 'stdin' && child) { + child.stdin.write(Buffer.from(message.data, 'base64')); + return; + } + if (message.type === 'end' && child) { child.stdin.end(); return; } + if (message.type === 'cancel') { terminate(); return; } + if (message.type !== 'start' || child) throw new Error('Expected start'); + if (typeof message.command !== 'string' || !Array.isArray(message.args)) throw new Error('Invalid command'); + const cwd = message.cwd ?? path.join(root, 'workspace'); + child = spawn(message.command, message.args, { + cwd, + env: { ...process.env, HOME: path.join(root, 'home'), ...message.env }, + detached: true, + stdio: ['pipe', 'pipe', 'pipe'], + }); + child.stdin.on('error', () => {}); + child.stdout.on('data', (data) => send(socket, { type: 'stdout', data: data.toString('base64') })); + child.stderr.on('data', (data) => send(socket, { type: 'stderr', data: data.toString('base64') })); + const finish = (code, signal, error) => { + if (done) return; + done = true; + clearTimeout(timer); + send(socket, { type: 'exit', code, signal, timedOut, error }); + socket.end(); + }; + child.on('error', (error) => finish(null, null, error.message)); + // Use exit, not close: a deliberately detached daemon can retain an fd. + child.on('exit', (code, signal) => finish(code, signal)); + if (message.timeoutMs > 0) timer = setTimeout(() => { timedOut = true; terminate(); }, message.timeoutMs); + }); + }); + server.listen(socketPath); +} else if (mode === 'call') { + const socket = net.connect(socketPath); + socket.on('error', (error) => { console.error(error.message); process.exitCode = 1; }); + socket.on('connect', () => process.stdin.pipe(socket, { end: false })); + socket.pipe(process.stdout); + socket.on('end', () => { process.stdin.pause(); process.stdin.unref?.(); }); +} else { + throw new Error('Invalid supervisor mode'); +} diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index eaae713b283..6ffe0998403 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -764,6 +764,10 @@ async function ensureSandboxCommandResolvable( throw new Error(`Timed out checking command "${command}" on sandbox target.`); } + if (target.providerKey === "exe-dev") { + throw new Error(`Compatible exe.dev image is missing ${command}; runtime installation is disabled.`); + } + // If the caller supplied an install command, attempt the install once via // the sandbox runner (which the sandbox provider wraps in a login shell) // and re-probe before reporting failure. This lets fresh sandbox leases @@ -1175,6 +1179,10 @@ export async function ensureAdapterExecutionTargetRuntimeCommandInstalled(input: } } + if (input.target.providerKey === "exe-dev") { + throw new Error(`Compatible exe.dev image is missing ${detectCommand ?? "the adapter runtime"}; runtime installation is disabled.`); + } + const result = await runAdapterExecutionTargetShellCommand( input.runId, input.target, @@ -1484,6 +1492,18 @@ export async function prepareAdapterExecutionTargetRuntime(input: { }; } + // A durable exe.dev workspace is seeded once. The marker is written only + // after preparation succeeds, so a failed first upload cannot make the next + // legacy run adopt an empty directory. Native runs supply their own verified + // checkpoint policy and never enter this branch. + const durableLegacyWorkspace = target.providerKey === "exe-dev" && input.workspaceInboundMode === undefined && input.syncWorkspace !== false; + const seedMarker = path.posix.join(input.workspaceRemoteDir ?? target.remoteCwd, ".paperclip-runtime", "workspace-seeded-v1"); + let inboundMode = input.workspaceInboundMode; + if (durableLegacyWorkspace) { + const marker = await requireSandboxRunner(target).execute({ command: "sh", args: ["-c", `test -f ${shellQuote(seedMarker)}`], timeoutMs: 15000 }); + if (marker.timedOut || (marker.exitCode !== 0 && marker.exitCode !== 1)) throw new Error("Could not inspect durable workspace seed"); + inboundMode = marker.exitCode === 0 ? "adopt_remote" : "host_current"; + } const prepared = await prepareCommandManagedRuntime({ runner: requireSandboxRunner(target), spec: { @@ -1500,7 +1520,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir: input.workspaceRemoteDir, syncWorkspace: input.syncWorkspace, - workspaceInboundMode: input.workspaceInboundMode, + workspaceInboundMode: inboundMode, workspaceDurableSeed: input.workspaceDurableSeed, workspaceBaseline: input.workspaceBaseline, workspaceGitSnapshot: input.workspaceGitSnapshot, @@ -1508,12 +1528,16 @@ export async function prepareAdapterExecutionTargetRuntime(input: { preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, additionalSources: input.additionalSources, - installCommand: input.installCommand, + installCommand: target.providerKey === "exe-dev" ? null : input.installCommand, detectCommand: input.detectCommand, onProgress: input.onProgress, onRuntimeProgress: input.onRuntimeProgress, runtimeSpan: input.runtimeSpan, }); + if (durableLegacyWorkspace) { + const marker = await requireSandboxRunner(target).execute({ command: "sh", args: ["-c", `mkdir -p ${shellQuote(path.posix.dirname(seedMarker))} && touch ${shellQuote(seedMarker)}`], timeoutMs: 15000 }); + if (marker.timedOut || marker.exitCode !== 0) throw new Error("Could not record durable workspace seed"); + } return { target, workspaceRemoteDir: prepared.workspaceRemoteDir, diff --git a/packages/plugins/sandbox-providers/exe-dev/README.md b/packages/plugins/sandbox-providers/exe-dev/README.md index 52d420c25da..db1be04c9aa 100644 --- a/packages/plugins/sandbox-providers/exe-dev/README.md +++ b/packages/plugins/sandbox-providers/exe-dev/README.md @@ -1,58 +1,128 @@ -# `@paperclipai/plugin-exe-dev` +# Experimental durable exe.dev environments -Published exe.dev sandbox provider plugin for Paperclip. +Enable **exe.dev Environments** in Instance → Experimental, install this bundled +plugin, and create a sandbox environment using **exe.dev VM (experimental)**. +Daytona remains the preferred Cloud default. Cloud installs this plugin only for +releases and stacks explicitly opted into `enableExeEnvironments`. -This package lives in the Paperclip monorepo, but it is intentionally excluded from the root `pnpm` workspace and shaped to publish and install like a standalone npm package. That lets operators install it from the Plugins page by package name without introducing root lockfile churn. +An environment owns one named VM. Each agent/workspace lease has a separate home, +working directory, socket, and systemd cgroup on that VM. Agents sharing a VM must +trust each other: these directories are not security boundaries. A VM cannot be +bound to two Paperclip companies or environments. -## Install +## Credentials and creation -From a Paperclip instance, install: +Register an SSH public key with exe.dev; save its matching private key in a +Paperclip company secret using the environment form. Management commands and VM +execution both use SSH, so an exe.dev HTTPS API token is unnecessary. An operator +may instead configure an absolute `sshIdentityFile`, or use +`EXE_DEV_SSH_PRIVATE_KEY` / `EXE_DEV_SSH_KEY_FILE` on a self-hosted controller. +Credentials remain on the controller and are never forwarded to the VM. +For a private image registry, set `registryAuth` to a company secret containing +`username:token` (or `EXE_DEV_REGISTRY_AUTH` for the explicit live harness). exe.dev +uses this credential to pull the image; agent commands do not receive it. -```text -@paperclipai/plugin-exe-dev +- **Attach:** specify `vmName` for an existing compatible VM. +- **Create:** specify an immutable `image@sha256:…` published from + `docker/exe-dev-runner/Dockerfile`. A stable name is derived from the company and + environment unless `vmName` is supplied. Retrying an uncertain create reconciles + that name. A connection test validates the account; the first run provisions + and verifies the image. An attached VM is inspected without modification. + +Arbitrary images and boot-time package installation are unsupported. The image +contains Node, systemd, the execution supervisor, the Rust runner, its matching +provider pack, and the supported legacy CLIs. Missing runtimes fail with an image +compatibility error. Existing 0.1 per-run configurations require explicit migration +to a compatible named VM; upgrading the plugin never deletes those old VMs. + +SSH uses a dedicated identity, disables agent forwarding, and rejects changed +host keys. `accept-new` persists first-use keys in +`~/.ssh/paperclip-exe-known_hosts`. For preverified keys, supply `knownHosts` entries +for both `exe.dev` and the VM; this forces strict checking. + +## Lifetime and concurrency + +Normal release retains the VM and reusable lease. Cancellation or lease destruction +stops only that lease's cgroup, waits for confirmation, and leaves files intact. +An SSH disconnection is not a termination receipt. A requested lease deadline +limits its systemd service; it never deletes the VM. Controller restart reattaches +to the same VM identity and disk. Missing or replaced VMs fail closed and require +explicit recovery into a new environment. + +The controller stores the resource identity in reserved environment metadata, +independently of run leases. Concurrent acquisitions must agree on that identity. +A durable marker on the VM detects replacement even if its DNS name is reused. +There is no automatic VM deletion, disk reclamation, or in-place image upgrade. +Disconnecting an environment retains its VM; delete it explicitly in exe.dev when +its data is no longer needed. + +Agent homes and workspaces live under `/var/lib/paperclip-exe///`. +Use the normal isolated-workspace policy for concurrent task work. Separate Git +worktrees avoid two agents modifying the same checkout. This feature does not add +a service registry or port allocator. Start a persistent web server as a separate +systemd service if it must survive agent cancellation; choose its port explicitly. + +VM previews must remain private and use exe.dev's own HTTPS sharing/authentication. +Native runners connect outward to a reachable `PAPERCLIP_RUNNER_PUBLIC_URL=wss://…`. +Cloud derives this origin from the stack's canonical hostname. Paperclip does not +proxy exe.dev preview pages. + +## Files and backups + +The first run seeds the remote workspace. Later runs adopt its persisted files; +legacy initialization is marked only after a successful seed. Completion uses the +existing conflict-aware Git/file copyback pipeline. Native runs additionally use +the existing verified harness/session checkpoint and restart-recovery contract. +The existing exclusions still apply, including dependencies, generated/cache +folders, ignored files, and runtime scratch. + +This is the same backup coverage as Daytona, not a disk image or continuous mirror +into object storage. In Cloud, copyback and native checkpoints under `PAPERCLIP_HOME` +land on the tenant's persistent volume, which has daily/weekly/monthly provider +backups. Explicitly uploaded artifacts use the separate object-storage path. +Files outside copied paths, live process memory, database consistency, ignored +files, and VM-wide state are not included. A controller filesystem outside the +backed-up volume needs its own backup policy. Restore is explicit and must be tested +against the data classes the application actually needs. + +## Build and verification + +```sh +pnpm --dir packages/plugins/sandbox-providers/exe-dev install --ignore-workspace +pnpm --dir packages/plugins/sandbox-providers/exe-dev build +pnpm --dir packages/plugins/sandbox-providers/exe-dev test +node cli/node_modules/tsx/dist/cli.mjs scripts/build-exe-dev-image.ts --push --metadata-file /tmp/exe-image.json ``` -## Configuration +The manual **Experimental exe.dev image** workflow publishes a content-addressed +Linux amd64 image and an immutable digest artifact. Production builds require a +clean checkout and frozen dependencies. `PAPERCLIP_IMAGE_ALLOW_DIRTY=1` is only for +explicit local development builds. -Configure exe.dev from `Instance Settings -> Environments`, not from the plugin's plugin page. +Run the explicit paid browser matrix and provider stress campaign: -- Put the exe.dev API token on the sandbox environment itself. -- When you save an environment, Paperclip stores pasted API keys and pasted SSH private keys as company secrets. -- `EXE_API_KEY` remains an optional host-level fallback when an environment omits the API token. -- The current implementation provisions VMs through exe.dev's HTTPS API and runs commands through direct SSH to the created VM. - -To use the provider successfully, the environment/host needs all of the following: - -- An exe.dev API token that allows the lifecycle commands the provider uses: `new`, `ls`, and `rm`. `whoami` and `help` are recommended for manual debugging. `restart` is only needed if you extend the provider to restart retained VMs. -- SSH access from the Paperclip host to the resulting `*.exe.xyz` VMs. -- An SSH private key that exe.dev already recognizes. You can either: - - paste the private key into the environment config via `sshPrivateKey` - - point `sshIdentityFile` at an absolute host path - - or leave both blank and rely on the host's default SSH agent/keychain -- The matching public key must already be registered with exe.dev before the provider can execute commands inside the VM. - -Operational notes: - -- If exe.dev replies `Please complete registration by running: ssh exe.dev`, the host key has not finished exe.dev onboarding yet. -- Reusable leases keep the VM alive between runs. exe.dev does not expose a documented "stop and later resume" command in the public CLI docs, so `reuseLease: true` means "retain the VM" rather than "suspend it." -- The provisioning path uses `https://exe.dev/exec`, which exe.dev documents as a command-style HTTPS API with a 30-second request timeout. Typical `new` calls are expected to fit inside that limit; command execution itself does not use `/exec`. -- Probes still create and delete a real exe.dev VM through `/exec`, and so do the `new`/`rm` calls inside the normal acquire/release lifecycle. Treat all of those as real provisioning cost, not just probes. -- exe.dev runs `--setup-script` as the unprivileged `exedev` user, not as root. That user has passwordless `sudo`, so any system-level steps in a custom `setupScript` must invoke `sudo` explicitly (for example `sudo apt-get install -y …`). When you omit `setupScript`, the plugin supplies a default that installs Node 24 via the official nodesource script — Paperclip's sandbox callback bridge is a Node program, so the VM needs `node` on `PATH` before the bridge can launch. - -## Local development - -```bash -cd packages/plugins/sandbox-providers/exe-dev -pnpm install --ignore-workspace --no-lockfile -pnpm build -pnpm test -pnpm typecheck +```sh +pnpm test:e2e:runner -- --suite exe-compatibility +pnpm test:e2e:runner -- --suite exe-recovery +EXE_DEV_LIVE_SMOKE=1 node cli/node_modules/tsx/dist/cli.mjs tests/exe-dev-live/stress.ts ``` -These commands assume the repo root has already been installed once so the local `@paperclipai/plugin-sdk` workspace package is available to the compiler during development. +Both require `EXE_DEV_SSH_PRIVATE_KEY` and `PAPERCLIP_E2E_EXE_IMAGE`; browser cells +also require their model credentials. Native browser cells need an externally +reachable WSS origin. For local testing, set `PAPERCLIP_E2E_RUNNER_TUNNEL_BIN` to an +installed `cloudflared` binary. The harness exposes only authenticated runner +WebSocket upgrades; all ordinary HTTP requests and board API paths return 404. +The matrix has seven profiles × three workflows. Stress +covers eight shared leases, concurrent commands, UTF-8/stdin, copyback, restart, +SSH loss, VM reboot, large output, escaped descendant cancellation, timeout, +cancellation isolation, and a separate private HTTP service. Set +`EXE_DEV_SOAK_MS` for an explicit idle soak. Campaign cleanup deletes only +campaign-owned VMs; production provider hooks never delete VMs. -## Package layout - -- `src/manifest.ts` declares the sandbox-provider driver metadata -- `src/plugin.ts` implements the environment lifecycle hooks -- `paperclipPlugin.manifest` and `paperclipPlugin.worker` point the host at the built plugin entrypoints in `dist/` +The native Codex message cell additionally runs two native and one legacy agent +concurrently on its VM and checks distinct homes/workspaces. The four-cell +recovery suite covers structured question/resume with and without a controller +restart for legacy and native Codex. `tests/exe-dev-live/preview.ts` provides an +interactive private Vite fixture with `create`, `cancel`, `edit`, and `cleanup` +commands; it deliberately keeps its VM until cleanup so a signed-in browser can +verify HMR and an idle soak. diff --git a/packages/plugins/sandbox-providers/exe-dev/package.json b/packages/plugins/sandbox-providers/exe-dev/package.json index e62831f4d78..b5cce5974a5 100644 --- a/packages/plugins/sandbox-providers/exe-dev/package.json +++ b/packages/plugins/sandbox-providers/exe-dev/package.json @@ -1,6 +1,6 @@ { "name": "@paperclipai/plugin-exe-dev", - "version": "0.1.2", + "version": "0.2.0", "description": "exe.dev sandbox provider plugin for Paperclip environments", "license": "MIT", "homepage": "https://github.com/paperclipai/paperclip", diff --git a/packages/plugins/sandbox-providers/exe-dev/src/manifest.ts b/packages/plugins/sandbox-providers/exe-dev/src/manifest.ts index bef86f3deff..e1e91a2e466 100644 --- a/packages/plugins/sandbox-providers/exe-dev/src/manifest.ts +++ b/packages/plugins/sandbox-providers/exe-dev/src/manifest.ts @@ -1,183 +1,39 @@ import type { PaperclipPluginManifestV1 } from "@paperclipai/plugin-sdk"; - -const PLUGIN_ID = "paperclip.exe-dev-sandbox-provider"; -const PLUGIN_VERSION = "0.1.2"; - const manifest: PaperclipPluginManifestV1 = { - id: PLUGIN_ID, - apiVersion: 1, - version: PLUGIN_VERSION, - displayName: "exe.dev Sandbox Provider", - description: - "Sandbox provider plugin that provisions exe.dev VMs as Paperclip execution environments.", - author: "Paperclip", - categories: ["automation"], - capabilities: ["environment.drivers.register"], - entrypoints: { - worker: "./dist/worker.js", - }, - environmentDrivers: [ - { - driverKey: "exe-dev", - kind: "sandbox_provider", - displayName: "exe.dev VM", - description: - "Provisions exe.dev VMs through the HTTPS API, then runs commands over direct SSH for long-lived Paperclip workloads.", - configSchema: { - type: "object", - properties: { - // ---- Essentials (always visible, in this order) ---- - apiKey: { - type: "string", - format: "secret-ref", - description: - "Paste your exe.dev API token, or pick a saved Paperclip secret. Create one at exe.dev → Settings → API tokens with `/exec` scope (`new`, `ls`, `rm`).", - }, - sshPrivateKey: { - type: "string", - format: "secret-ref", - maxLength: 8192, - description: - "Paste the SSH private key you registered with exe.dev, or pick a saved secret. Leave blank to fall back to an on-host key (see Advanced → SSH access).", - }, - // ---- Advanced: SSH access ---- - sshUser: { - type: "string", - description: - "Login user on the VM. Leave blank to use the image default, usually `root`.", - "x-paperclip-advanced": true, - "x-paperclip-group": "SSH access", - }, - sshIdentityFile: { - type: "string", - description: - "Absolute path to a private key on the Paperclip host. Used only when SSH Private Key is empty.", - "x-paperclip-advanced": true, - "x-paperclip-group": "SSH access", - }, - sshPort: { - type: "number", - description: "SSH port for direct VM access.", - default: 22, - "x-paperclip-advanced": true, - "x-paperclip-group": "SSH access", - }, - strictHostKeyChecking: { - type: "string", - description: - "Host key policy passed to ssh via StrictHostKeyChecking. Typical values are `accept-new`, `yes`, or `no`.", - default: "accept-new", - "x-paperclip-advanced": true, - "x-paperclip-group": "SSH access", - }, - // ---- Advanced: VM resources ---- - image: { - type: "string", - description: "Optional container image to use when creating the VM.", - "x-paperclip-advanced": true, - "x-paperclip-group": "VM resources", - }, - cpu: { - type: "number", - description: "Optional CPU count passed to `exe.dev new --cpu`.", - default: 4, - "x-paperclip-advanced": true, - "x-paperclip-group": "VM resources", - }, - memory: { - type: "string", - description: "Optional memory size such as `4GB`.", - default: "4GB", - "x-paperclip-advanced": true, - "x-paperclip-group": "VM resources", - }, - disk: { - type: "string", - description: "Optional disk size such as `20GB`.", - default: "20GB", - "x-paperclip-advanced": true, - "x-paperclip-group": "VM resources", - }, - // ---- Advanced: VM creation ---- - command: { - type: "string", - description: "Optional container command passed to `exe.dev new --command`.", - "x-paperclip-advanced": true, - "x-paperclip-group": "VM creation", - }, - env: { - type: "object", - description: "Optional environment variables applied at VM creation time.", - additionalProperties: { type: "string" }, - "x-paperclip-advanced": true, - "x-paperclip-group": "VM creation", - }, - integrations: { - type: "array", - description: "Optional exe.dev integrations to attach during VM creation.", - items: { type: "string" }, - "x-paperclip-advanced": true, - "x-paperclip-group": "VM creation", - }, - tags: { - type: "array", - description: "Optional tags to apply during VM creation.", - items: { type: "string" }, - "x-paperclip-advanced": true, - "x-paperclip-group": "VM creation", - }, - setupScript: { - type: "string", - description: "Optional first-boot setup script passed to `exe.dev new --setup-script`.", - "x-paperclip-advanced": true, - "x-paperclip-group": "VM creation", - }, - prompt: { - type: "string", - description: "Optional Shelley prompt passed to `exe.dev new --prompt`.", - "x-paperclip-advanced": true, - "x-paperclip-group": "VM creation", - }, - comment: { - type: "string", - description: "Optional short note attached to created VMs.", - "x-paperclip-advanced": true, - "x-paperclip-group": "VM creation", - }, - namePrefix: { - type: "string", - description: "Optional prefix used when generating VM names.", - default: "paperclip", - "x-paperclip-advanced": true, - "x-paperclip-group": "VM creation", - }, - // ---- Advanced: API + runtime ---- - apiUrl: { - type: "string", - description: - "Optional exe.dev HTTPS API base URL or /exec endpoint. Defaults to https://exe.dev/exec.", - "x-paperclip-advanced": true, - "x-paperclip-group": "API + runtime", - }, - timeoutMs: { - type: "number", - description: "Timeout for VM lifecycle and SSH operations in milliseconds.", - default: 300000, - "x-paperclip-advanced": true, - "x-paperclip-group": "API + runtime", - }, - reuseLease: { - type: "boolean", - description: - "Whether to keep the VM alive between runs instead of deleting it on release.", - default: false, - "x-paperclip-advanced": true, - "x-paperclip-group": "API + runtime", - }, - }, + id: "paperclip.exe-dev-sandbox-provider", apiVersion: 1, version: "0.2.0", + displayName: "exe.dev (experimental)", + description: "Durable, private exe.dev VMs shared by a trusted group of agents.", + author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], + entrypoints: { worker: "./dist/worker.js" }, + environmentDrivers: [{ + driverKey: "exe-dev", kind: "sandbox_provider", displayName: "exe.dev VM (experimental)", + description: "One persistent VM per environment; independent agent homes and workspaces. Disconnecting keeps the VM.", + supportsReusableLeases: true, + sandboxCapabilities: { + reusableLeases: true, persistentProcessSessions: true, independentControlCommands: true, + incrementalSessionOutput: true, duplexCommandStream: true, + nativeSyncIn: false, nativeSyncOut: false, concurrentSyncOperations: false, runnerWebSocketIngress: false, + }, + configSchema: { + type: "object", + properties: { + mode: { type: "string", enum: ["attach", "create"], default: "attach", description: "Attach a compatible VM or create one from a pinned Paperclip image." }, + vmName: { type: "string", description: "Durable VM name. Required for attach; create derives a stable name when omitted." }, + sshPrivateKey: { type: "string", format: "secret-ref", maxLength: 8192, description: "Private SSH key registered with exe.dev. Stored as a Paperclip secret." }, + image: { type: "string", description: "Published Paperclip exe.dev image with @sha256 digest. Required to create; never installed or upgraded at boot." }, + registryAuth: { type: "string", format: "secret-ref", description: "Optional private registry credential in username:token format, used by exe.dev only while pulling the VM image." }, + knownHosts: { type: "string", description: "Optional verified OpenSSH known_hosts entries for exe.dev and the VM. Pins host keys with strict checking." }, + sshIdentityFile: { type: "string", description: "Optional absolute key path on the Paperclip host.", "x-paperclip-advanced": true }, + strictHostKeyChecking: { type: "string", enum: ["yes", "accept-new"], default: "accept-new", description: "First-use keys are persisted on the controller; changed keys are always rejected.", "x-paperclip-advanced": true }, + cpu: { type: "integer", minimum: 1, default: 2, "x-paperclip-advanced": true }, + memory: { type: "string", default: "4GB", "x-paperclip-advanced": true }, + disk: { type: "string", default: "20GB", "x-paperclip-advanced": true }, + timeoutMs: { type: "integer", minimum: 1000, default: 300000, "x-paperclip-advanced": true }, + reuseLease: { type: "boolean", default: true, description: "Reuse each agent workspace between runs. The VM is always retained." }, + runnerLifecycleMode: { type: "string", enum: ["inherit", "per_turn", "warm"], default: "inherit" }, + runnerIdleTimeoutMs: { type: "integer", minimum: 1000, maximum: 86400000, default: 300000 }, }, }, - ], + }], }; - export default manifest; diff --git a/packages/plugins/sandbox-providers/exe-dev/src/plugin.test.ts b/packages/plugins/sandbox-providers/exe-dev/src/plugin.test.ts index 16618bf5073..34f5ee47a75 100644 --- a/packages/plugins/sandbox-providers/exe-dev/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/exe-dev/src/plugin.test.ts @@ -1,870 +1,149 @@ -import { EventEmitter } from "node:events"; import { beforeEach, describe, expect, it, vi } from "vitest"; - -const fetchMock = vi.fn(); -const spawnMock = vi.hoisted(() => vi.fn()); - -vi.stubGlobal("fetch", fetchMock); - -vi.mock("node:child_process", async () => { - const actual = await vi.importActual("node:child_process"); - return { - ...actual, - spawn: spawnMock, - }; -}); - -import plugin, { validateSshPrivateKey } from "./plugin.js"; +const transport = vi.hoisted(() => ({ ssh: vi.fn(), openSsh: vi.fn() })); +vi.mock("./transport.js", () => ({ ...transport, quote: (value: string) => `'${value.replace(/'/g, `'"'"'`)}'` })); +import plugin, { parseConfig, validateSshPrivateKey } from "./plugin.js"; import manifest from "./manifest.js"; - -class MockChildProcess extends EventEmitter { - stdout = new EventEmitter(); - stderr = new EventEmitter(); - stdin = { - written: "" as string, - ended: false, - write: (chunk: string) => { - this.stdin.written += chunk; - return true; - }, - end: () => { - this.stdin.ended = true; - }, - }; - kill = vi.fn(); - - constructor(input: { code?: number; signal?: string | null; stdout?: string; stderr?: string }) { - super(); - queueMicrotask(() => { - if (input.stdout) this.stdout.emit("data", input.stdout); - if (input.stderr) this.stderr.emit("data", input.stderr); - this.emit("close", input.code ?? 0, input.signal ?? null); - }); - } -} - -function queueSpawnResult(input: { code?: number; signal?: string | null; stdout?: string; stderr?: string }) { - spawnMock.mockImplementationOnce(() => new MockChildProcess(input)); -} - -describe("exe.dev sandbox provider plugin", () => { - beforeEach(() => { - fetchMock.mockReset(); - spawnMock.mockReset(); - delete process.env.EXE_API_KEY; +const hooks = plugin.definition; +const config = { reuseLease: true, mode: "attach", vmName: "test-box", sshIdentityFile: "/keys/exe" }; +const params = { driverKey: "exe-dev", companyId: "company", environmentId: "environment", config }; +const bindingId = "00000000-0000-4000-8000-000000000001"; +const image = { schema: 1, sourceRevision: "abc123", contentId: "content" }; +let vmExists = true; +let privateVm = true; +let scope = ""; +beforeEach(async () => { + await hooks.onShutdown?.(); + vmExists = true; privateVm = true; scope = ""; + transport.ssh.mockReset(); transport.openSsh.mockReset(); + transport.ssh.mockImplementation(async (_config, host, command) => { + if (host === "exe.dev") { + if (command.startsWith("ls")) return JSON.stringify({ vms: vmExists ? [{ vm_name: "test-box", proxy_share: privateVm ? "private" : "public" }] : [] }); + if (command.startsWith("new")) { vmExists = true; return "{}"; } + throw Error(`Unexpected management command: ${command}`); + } + if (command.includes("image.json")) return JSON.stringify(image); + if (command.includes("paperclip-exe-claim.lock")) { + scope = command.match(/scope:[^a-f0-9]*([a-f0-9]{32})/)?.[1] ?? ""; + return bindingId; + } + if (command.includes("cat /var/lib/paperclip-exe/binding.json")) return JSON.stringify({ id: bindingId, scope }); + return ""; }); +}); +const acquire = (runId = "run") => hooks.onEnvironmentAcquireLease!({ ...params, runId, agentId: runId }); - it("declares environment lifecycle handlers", async () => { - expect(await plugin.definition.onHealth?.()).toEqual({ - status: "ok", - message: "exe.dev sandbox provider plugin healthy", - }); - expect(plugin.definition.onEnvironmentAcquireLease).toBeTypeOf("function"); - expect(plugin.definition.onEnvironmentExecute).toBeTypeOf("function"); +describe("durable exe.dev lifecycle", () => { + it("allocates independent leases on one VM without provisioning per run", async () => { + const [a, b] = await Promise.all([acquire("a"), acquire("b")]); + expect(a.providerLeaseId).not.toBe(b.providerLeaseId); + expect(a.metadata?.vmName).toBe("test-box"); + expect(a.metadata?.remoteCwd).not.toBe(b.metadata?.remoteCwd); + expect(a.metadata?.remoteHome).not.toBe(b.metadata?.remoteHome); + expect(transport.ssh.mock.calls.some(([, host, cmd]) => host === "exe.dev" && !cmd.startsWith("ls"))).toBe(false); }); - - it("normalizes config and emits SSH guidance warnings", async () => { - process.env.EXE_API_KEY = "host-key"; - - const result = await plugin.definition.onEnvironmentValidateConfig?.({ - driverKey: "exe-dev", - config: { - apiUrl: "https://exe.dev", - namePrefix: " Paperclip Sandbox ", - image: " ubuntu:22.04 ", - cpu: "4.8", - memory: " 8GB ", - disk: " 50GB ", - env: { - FOO: " bar ", - }, - integrations: [" github "], - tags: "prod, sandbox", - timeoutMs: "450000.9", - reuseLease: true, - sshPort: "2222", - }, - }); - - expect(result).toEqual({ - ok: true, - warnings: [ - "The Paperclip host must have SSH access to the created exe.dev VM, and its SSH key must be registered with exe.dev. The API token only covers provisioning.", - "reuseLease keeps the VM alive between runs; this provider does not suspend retained VMs.", - ], - normalizedConfig: { - apiKey: null, - apiUrl: "https://exe.dev/exec", - namePrefix: "paperclip-sandbox", - image: "ubuntu:22.04", - command: null, - cpu: 4, - memory: "8GB", - disk: "50GB", - comment: null, - env: { FOO: "bar" }, - integrations: ["github"], - tags: ["prod", "sandbox"], - setupScript: null, - prompt: null, - timeoutMs: 450000, - reuseLease: true, - sshUser: null, - sshPrivateKey: null, - sshIdentityFile: null, - sshPort: 2222, - strictHostKeyChecking: "accept-new", - }, - }); + it("normal release retains the VM and warm processes", async () => { + const lease = await acquire(); transport.ssh.mockClear(); + await hooks.onEnvironmentReleaseLease!({ ...params, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata }); + expect(transport.ssh).not.toHaveBeenCalled(); }); - - it("normalizes trailing /exec apiUrl inputs without duplication", async () => { - process.env.EXE_API_KEY = "host-key"; - - const result = await plugin.definition.onEnvironmentValidateConfig?.({ - driverKey: "exe-dev", - config: { - apiUrl: "https://exe.dev/exec/", - }, - }); - - expect(result).toMatchObject({ - ok: true, - normalizedConfig: { - apiUrl: "https://exe.dev/exec", - }, - }); + it("destroy confirms only the lease cgroup stopped and never deletes a VM", async () => { + const lease = await acquire(); transport.ssh.mockClear(); + const receipt = await hooks.onEnvironmentDestroyLease!({ ...params, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata }); + expect(receipt).toEqual({ providerLeaseId: lease.providerLeaseId, state: "stopped" }); + expect(transport.ssh.mock.calls.some(([, , cmd]) => cmd.includes("systemctl stop"))).toBe(true); + expect(transport.ssh.mock.calls.some(([, host]) => host === "exe.dev")).toBe(false); }); - - it("rejects invalid config", async () => { - await expect(plugin.definition.onEnvironmentValidateConfig?.({ - driverKey: "exe-dev", - config: { - apiUrl: "not-a-url", - cpu: 0, - env: { - "BAD-KEY": "value", - }, - sshPort: 70000, - strictHostKeyChecking: "", - timeoutMs: 0, - }, - })).resolves.toEqual({ - ok: false, - warnings: [ - "The Paperclip host must have SSH access to the created exe.dev VM, and its SSH key must be registered with exe.dev. The API token only covers provisioning.", - ], - errors: [ - "apiUrl must be a valid URL.", - "timeoutMs must be between 1 and 86400000.", - "cpu must be greater than 0 when provided.", - "sshPort must be between 1 and 65535.", - "exe.dev environments require an API key in config or EXE_API_KEY.", - "env contains an invalid key: BAD-KEY", - "strictHostKeyChecking cannot be empty.", - ], - }); + it("does not issue a termination receipt after transport loss", async () => { + const lease = await acquire(); + transport.ssh.mockRejectedValueOnce(new Error("connection lost")); + await expect(hooks.onEnvironmentDestroyLease!({ ...params, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata })).rejects.toThrow("connection lost"); }); - - describe("sshPrivateKey validation", () => { - const VALID_OPENSSH = [ - "-----BEGIN OPENSSH PRIVATE KEY-----", - "b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gt", - "ZWQyNTUxOQAAACBPzMxQp4Y6XCfDV2t6oWmqHkKx0K7C7w7q9F6gQ3jPbgAAAJjJ8jjE", - "yfI4xAAAAAtzc2gtZWQyNTUxOQAAACBPzMxQp4Y6XCfDV2t6oWmqHkKx0K7C7w7q9F6g", - "Q3jPbgAAAEDqLhB4kV1tw8m4gE9oNCkF2cJv0YnHQ8E5sHU3xKnD5k/MzFCnhjpcJ8NX", - "a3qhaaoeQrHQrsLvDur0XqBDeM9uAAAAFXVzZXJAaG9zdAECAwQ=", - "-----END OPENSSH PRIVATE KEY-----", - ].join("\n"); - const VALID_RSA_PEM = [ - "-----BEGIN RSA PRIVATE KEY-----", - "MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu", - "KUpRKfFLfRYC9AIKjbJTWit+CqvjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm", - "o3qGy0t6z5tZbcgvflRslzu1HxXLpwYqQq2gMNw9UQAoHs3rDl+EzBjF6trBV5wF", - "wQIhANwiwDR7TVlIRk5kbgPMd2dDgY8mAU1cQ8KbWvjVMmKxAiEAxYTUyVjwhfQy", - "VJoR7T0n4XdR1n+W8Eth7AEPxnHfaQECIB5cNuqB9F1qC2pSyf6e+UAyl9rmKQXp", - "-----END RSA PRIVATE KEY-----", - ].join("\n"); - - it("accepts a valid OpenSSH PEM block", () => { - expect(validateSshPrivateKey(VALID_OPENSSH)).toBeNull(); - }); - - it("accepts a valid PKCS#1 RSA PEM block", () => { - expect(validateSshPrivateKey(VALID_RSA_PEM)).toBeNull(); - }); - - it("accepts UUID-like secret reference values from the save-time schema stage", async () => { - process.env.EXE_API_KEY = "host-key"; - - const result = await plugin.definition.onEnvironmentValidateConfig?.({ - driverKey: "exe-dev", - config: { - apiKey: "api-key", - sshPrivateKey: "11111111-1111-4111-8111-111111111111", - }, - }); - - expect(result).toMatchObject({ - ok: true, - normalizedConfig: { - sshPrivateKey: "11111111-1111-4111-8111-111111111111", - }, - }); - expect(result?.errors ?? []).toEqual([]); - }); - - it("treats empty / whitespace-only input as valid (falls back to on-host key)", () => { - expect(validateSshPrivateKey("")).toBeNull(); - expect(validateSshPrivateKey(" \n\n ")).toBeNull(); - }); - - it("rejects a pasted public key", () => { - expect( - validateSshPrivateKey("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE+gT9 user@host"), - ).toMatch(/looks like a PUBLIC key/); - }); - - it("rejects a PuTTY PPK file paste", () => { - const ppk = [ - "PuTTY-User-Key-File-3: ssh-ed25519", - "Encryption: none", - "Comment: imported-openssh-key", - "Public-Lines: 2", - "AAAAC3NzaC1lZDI1NTE5AAAAIE+gT9zMxQp4Y6XCfDV2t6oWmqHkKx0K7C7w7q9F6g", - "Q3jP", - ].join("\n"); - expect(validateSshPrivateKey(ppk)).toMatch(/PuTTY \.ppk/); - }); - - it("rejects a missing END marker (truncated paste)", () => { - const truncated = VALID_OPENSSH.split("\n").slice(0, -1).join("\n"); - expect(validateSshPrivateKey(truncated)).toMatch(/missing its '-----END/); - }); - - it("rejects a body with non-base64 characters", () => { - const garbled = [ - "-----BEGIN OPENSSH PRIVATE KEY-----", - "this is not base64!!", - "-----END OPENSSH PRIVATE KEY-----", - ].join("\n"); - expect(validateSshPrivateKey(garbled)).toMatch(/non-base64/); - }); - - it("rejects a header/footer label mismatch", () => { - const mismatched = [ - "-----BEGIN OPENSSH PRIVATE KEY-----", - "Zm9vYmFy", - "-----END RSA PRIVATE KEY-----", - ].join("\n"); - expect(validateSshPrivateKey(mismatched)).toMatch(/header\/footer mismatch/); - }); - - it("returns the sshPrivateKey error from onEnvironmentValidateConfig on save", async () => { - process.env.EXE_API_KEY = "host-key"; - - const result = await plugin.definition.onEnvironmentValidateConfig?.({ - driverKey: "exe-dev", - config: { - sshPrivateKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE+gT9 user@host", - }, - }); - - expect(result?.ok).toBe(false); - expect(result?.errors ?? []).toEqual( - expect.arrayContaining([expect.stringMatching(/sshPrivateKey looks like a PUBLIC key/)]), - ); - }); + it("rejects a foreign-company lease before contacting the VM", async () => { + const lease = await acquire(); transport.ssh.mockClear(); + await expect(hooks.onEnvironmentDestroyLease!({ ...params, companyId: "other", providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata })).rejects.toThrow("does not belong"); + expect(transport.ssh).not.toHaveBeenCalled(); }); - - it("acquires a lease by creating a VM and preparing the SSH workspace", async () => { - fetchMock.mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-env-run", - ssh_dest: "paperclip-env-run.exe.xyz", - https_url: "https://paperclip-env-run.exe.xyz", - status: "running", - }), { status: 200 }), - ); - queueSpawnResult({ stdout: "/home/exe\nbash\n" }); - queueSpawnResult({}); - - const lease = await plugin.definition.onEnvironmentAcquireLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - runId: "run-1", - requestedCwd: "/workspace/custom", - config: { - apiKey: "api-key", - namePrefix: "paperclip", - image: "ubuntu:22.04", - timeoutMs: 300000, - }, - }); - - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(String(fetchMock.mock.calls[0]?.[1]?.body ?? "")).toContain("new --json --no-email"); - expect(spawnMock).toHaveBeenCalledTimes(2); - expect(lease).toMatchObject({ - providerLeaseId: "paperclip-env-run", - metadata: { - provider: "exe-dev", - vmName: "paperclip-env-run", - sshDest: "paperclip-env-run.exe.xyz", - remoteCwd: "/workspace/custom", - shellCommand: "bash", - reuseLease: false, - }, - }); + it("rejects replacement of a VM with the same DNS name", async () => { + const lease = await acquire(); + const original = transport.ssh.getMockImplementation()!; + transport.ssh.mockImplementation((...args) => String(args[2]).includes("cat /var/lib/paperclip-exe/binding.json") ? JSON.stringify({ id: "replacement", scope }) : original(...args)); + await expect(hooks.onEnvironmentResumeLease!({ ...params, providerLeaseId: lease.providerLeaseId!, leaseMetadata: lease.metadata })).rejects.toThrow("identity changed"); }); - - it("uses a pasted sshPrivateKey when connecting to the VM", async () => { - fetchMock.mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-env-run", - ssh_dest: "paperclip-env-run.exe.xyz", - https_url: "https://paperclip-env-run.exe.xyz", - status: "running", - }), { status: 200 }), - ); - queueSpawnResult({ stdout: "/home/exe\nbash\n" }); - queueSpawnResult({}); - - await plugin.definition.onEnvironmentAcquireLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - runId: "run-1", - config: { - apiKey: "api-key", - sshPrivateKey: "-----BEGIN PRIVATE KEY-----\npretend\n-----END PRIVATE KEY-----", - }, - }); - - const firstSpawnArgs = spawnMock.mock.calls[0]?.[1] as string[] | undefined; - expect(firstSpawnArgs).toContain("-i"); - expect(firstSpawnArgs).toContain("-o"); - expect(firstSpawnArgs).toContain("IdentitiesOnly=yes"); + it("resumes the same workspace and home", async () => { + const lease = await acquire(); + const resumed = await hooks.onEnvironmentResumeLease!({ ...params, providerLeaseId: lease.providerLeaseId!, leaseMetadata: lease.metadata }); + expect(resumed.providerLeaseId).toBe(lease.providerLeaseId); + expect(resumed.metadata?.remoteCwd).toBe(lease.metadata?.remoteCwd); + expect(resumed.metadata?.resumedLease).toBe(true); }); - - it("supplies a default Node-install setup script when none is provided", async () => { - fetchMock.mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-env-run", - ssh_dest: "paperclip-env-run.exe.xyz", - https_url: "https://paperclip-env-run.exe.xyz", - status: "running", - }), { status: 200 }), - ); - queueSpawnResult({ stdout: "/home/exedev\nbash\n" }); - queueSpawnResult({}); - - await plugin.definition.onEnvironmentAcquireLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - runId: "run-1", - config: { - apiKey: "api-key", - }, - }); - - const body = String(fetchMock.mock.calls[0]?.[1]?.body ?? ""); - expect(body).toContain("--setup-script="); - expect(body).toContain("process.versions.node"); - expect(body).toContain("v[0]>24||(v[0]===24&&v[1]>=11)"); - expect(body).toContain("nodesource.com/setup_24.x"); - expect(body).toContain("sudo apt-get install -y nodejs"); + it("probes without allocating, claiming, or deleting resources", async () => { + expect((await hooks.onEnvironmentProbe!(params)).ok).toBe(true); + expect(transport.ssh).toHaveBeenCalledTimes(2); + expect(transport.ssh.mock.calls.some(([, , cmd]) => /new |rm |flock|systemd-run/.test(cmd))).toBe(false); }); - - it("preserves an operator-supplied setup script and does not append the default", async () => { - fetchMock.mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-env-run", - ssh_dest: "paperclip-env-run.exe.xyz", - https_url: "https://paperclip-env-run.exe.xyz", - status: "running", - }), { status: 200 }), - ); - queueSpawnResult({ stdout: "/home/exedev\nbash\n" }); - queueSpawnResult({}); - - await plugin.definition.onEnvironmentAcquireLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - runId: "run-1", - config: { - apiKey: "api-key", - setupScript: "echo custom", - }, - }); - - const body = String(fetchMock.mock.calls[0]?.[1]?.body ?? ""); - expect(body).toContain("--setup-script='echo custom'"); - expect(body).not.toContain("nodesource.com"); + it("does not provision in response to a missing-VM probe", async () => { + vmExists = false; + expect((await hooks.onEnvironmentProbe!(params)).ok).toBe(false); + expect(transport.ssh).toHaveBeenCalledTimes(1); }); - - it("does not redact the built-in default setup script in API errors", async () => { - fetchMock.mockResolvedValueOnce(new Response("upstream boom", { status: 500 })); - - const acquirePromise = plugin.definition.onEnvironmentAcquireLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - runId: "run-1", - config: { - apiKey: "api-key", - }, - }); - - await expect(acquirePromise).rejects.toMatchObject({ - name: "ExeDevApiError", - status: 500, - }); - - await acquirePromise?.catch((error: Error) => { - // Operator did not supply a setupScript, so the visible default install - // is not a secret and stays in the error for debuggability. - expect(error.message).toContain("nodesource.com/setup_24.x"); - expect(error.message).not.toContain("[REDACTED]"); - }); + it("rejects publicly shared preview URLs", async () => { + privateVm = false; + await expect(acquire()).rejects.toThrow("private HTTP"); }); - - it("surfaces exe.dev SSH onboarding guidance during lease acquisition", async () => { - fetchMock.mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-env-run", - ssh_dest: "paperclip-env-run.exe.xyz", - https_url: "https://paperclip-env-run.exe.xyz", - status: "running", - }), { status: 200 }), - ); - fetchMock.mockResolvedValueOnce(new Response("{}", { status: 200 })); - queueSpawnResult({ code: 1, stdout: "Please complete registration by running: ssh exe.dev\n" }); - - await expect(plugin.definition.onEnvironmentAcquireLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - runId: "run-1", - config: { - apiKey: "api-key", - timeoutMs: 300000, - }, - })).rejects.toThrow( - "the Paperclip host SSH key is not registered with exe.dev", - ); - - expect(String(fetchMock.mock.calls[1]?.[1]?.body ?? "")).toBe("rm --json 'paperclip-env-run'"); + it("rechecks private sharing on resume", async () => { + const lease = await acquire(); privateVm = false; + await expect(hooks.onEnvironmentResumeLease!({ ...params, providerLeaseId: lease.providerLeaseId!, leaseMetadata: lease.metadata })).rejects.toThrow("private HTTP"); }); - - it("surfaces invalid SSH key-format guidance during lease acquisition", async () => { - fetchMock.mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-env-run", - ssh_dest: "paperclip-env-run.exe.xyz", - https_url: "https://paperclip-env-run.exe.xyz", - status: "running", - }), { status: 200 }), - ); - fetchMock.mockResolvedValueOnce(new Response("{}", { status: 200 })); - queueSpawnResult({ - code: 255, - stderr: 'Load key "/tmp/paperclip-exe-dev-ssh-abc/id_ed25519": invalid format\n', - }); - - await expect(plugin.definition.onEnvironmentAcquireLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - runId: "run-1", - config: { - apiKey: "api-key", - sshPrivateKey: "not-actually-a-key", - timeoutMs: 300000, - }, - })).rejects.toThrow( - "the configured SSH private key isn't an OpenSSH-format private key", - ); - - expect(String(fetchMock.mock.calls[1]?.[1]?.body ?? "")).toBe("rm --json 'paperclip-env-run'"); + it("never replaces a missing VM after a durable identity is recorded", async () => { + const lease = await acquire(); vmExists = false; transport.ssh.mockClear(); + await expect(hooks.onEnvironmentAcquireLease!({ ...params, runId: "replacement", config: { ...config, mode: "create", image: `ghcr.io/example/exe@sha256:${"a".repeat(64)}` }, resourceBinding: lease.metadata?.environmentResourceBinding as any })).rejects.toThrow("no automatic replacement"); + expect(transport.ssh.mock.calls.every(([, , cmd]) => cmd.startsWith("ls"))).toBe(true); }); - - it("redacts sensitive lifecycle flags in API errors", async () => { - fetchMock.mockResolvedValueOnce(new Response("upstream boom", { status: 500 })); - - const acquirePromise = plugin.definition.onEnvironmentAcquireLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - runId: "run-1", - config: { - apiKey: "api-key", - env: { - SECRET: "super-secret", - }, - prompt: "build me a secret app", - setupScript: "export TOKEN=super-secret", - }, - }); - - await expect(acquirePromise).rejects.toMatchObject({ - name: "ExeDevApiError", - status: 500, - body: "upstream boom", - }); - - await acquirePromise?.catch((error: Error) => { - expect(error.message).toContain("--env='SECRET=[REDACTED]'"); - expect(error.message).toContain("--prompt='[REDACTED]'"); - expect(error.message).toContain("--setup-script='[REDACTED]'"); - expect(error.message).not.toContain("super-secret"); - }); + it("creates from a digest and reconciles the stable name", async () => { + vmExists = false; + await hooks.onEnvironmentAcquireLease!({ ...params, runId: "a", config: { ...config, mode: "create", image: `ghcr.io/example/exe@sha256:${"a".repeat(64)}` } }); + const command = transport.ssh.mock.calls.find(([, host, cmd]) => host === "exe.dev" && cmd.startsWith("new"))?.[2]; + expect(command).toContain("--name='test-box'"); + expect(command).not.toContain("setup-script"); + expect(command).not.toContain("prompt"); }); - - it("returns an expired lease when the retained VM no longer exists", async () => { - fetchMock.mockResolvedValueOnce( - new Response(JSON.stringify({ vms: [] }), { status: 200 }), - ); - - const lease = await plugin.definition.onEnvironmentResumeLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - providerLeaseId: "missing-vm", - config: { - apiKey: "api-key", - }, - leaseMetadata: { - sshDest: "missing-vm.exe.xyz", - }, - }); - - expect(lease).toEqual({ - providerLeaseId: null, - metadata: { - expired: true, - }, - }); + it("preserves the image digest when the host overlays lease metadata onto configuration", async () => { + const createConfig = { ...config, mode: "create", image: `ghcr.io/example/exe@sha256:${"a".repeat(64)}` }; + const lease = await hooks.onEnvironmentAcquireLease!({ ...params, config: createConfig, runId: "a" }); + const realized = await hooks.onEnvironmentRealizeWorkspace!({ ...params, config: { ...createConfig, ...lease.metadata }, lease, workspace: {} }); + expect(realized.cwd).toBe(lease.metadata?.remoteCwd); + expect(lease.metadata?.exeImageProvenance).toEqual(image); }); - - it("executes commands over SSH with cwd, env, and stdin", async () => { - queueSpawnResult({ code: 0, stdout: "hello\n", stderr: "" }); - - const result = await plugin.definition.onEnvironmentExecute?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - config: { - apiKey: "api-key", - timeoutMs: 300000, - }, - lease: { - providerLeaseId: "vm-1", - metadata: { - sshDest: "vm-1.exe.xyz", - }, - }, - command: "node", - args: ["-e", "process.stdout.write('hello\\n')"], - cwd: "/workspace", - env: { - FOO: "bar", - }, - stdin: "input-body", - timeoutMs: 1000, - }); - - expect(spawnMock).toHaveBeenCalledTimes(1); - expect(spawnMock.mock.calls[0]?.[0]).toBe("ssh"); - const remoteScript = String(spawnMock.mock.calls[0]?.[1]?.at(-1) ?? ""); - expect(remoteScript).toContain("/workspace"); - expect(remoteScript).toContain("FOO='"); - // The wrapper sources no `nvm.sh`; the sandbox image supplies node on PATH. - expect(remoteScript).not.toContain("nvm.sh"); - expect(remoteScript).not.toContain("NVM_DIR"); - const child = spawnMock.mock.results[0]?.value as MockChildProcess; - expect(child.stdin.written).toBe("input-body"); - expect(child.stdin.ended).toBe(true); - expect(result).toMatchObject({ - exitCode: 0, - timedOut: false, - stdout: "hello\n", - stderr: "", - metadata: { - provider: "exe-dev", - vmName: "vm-1", - }, - }); - }); - - it("returns exe.dev SSH onboarding guidance for command execution failures", async () => { - queueSpawnResult({ code: 1, stdout: "Please complete registration by running: ssh exe.dev\n" }); - - const result = await plugin.definition.onEnvironmentExecute?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - config: { - apiKey: "api-key", - timeoutMs: 300000, - }, - lease: { - providerLeaseId: "vm-1", - metadata: { - sshDest: "vm-1.exe.xyz", - }, - }, - command: "node", - args: ["-v"], - }); - - expect(result?.exitCode).toBe(1); - expect(String(result?.stderr ?? "")).toContain("the Paperclip host SSH key is not registered with exe.dev"); - expect(String(result?.stderr ?? "")).toContain("ssh exe.dev"); - }); - - it("probes by creating and then deleting a VM after SSH verification", async () => { - fetchMock - .mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-probe", - ssh_dest: "paperclip-probe.exe.xyz", - status: "running", - }), { status: 200 }), - ) - .mockResolvedValueOnce(new Response("{}", { status: 200 })); - queueSpawnResult({ stdout: "/home/exe\nbash\n" }); - queueSpawnResult({}); - - const result = await plugin.definition.onEnvironmentProbe?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - config: { - apiKey: "api-key", - }, - }); - - expect(result).toMatchObject({ - ok: true, - summary: "Connected to exe.dev VM paperclip-probe.", - metadata: { - provider: "exe-dev", - vmName: "paperclip-probe", - sshDest: "paperclip-probe.exe.xyz", - shellCommand: "bash", - }, - }); - expect(String(fetchMock.mock.calls[1]?.[1]?.body ?? "")).toBe("rm --json 'paperclip-probe'"); - }); - - it("cleans up the probe VM when SSH verification fails", async () => { - fetchMock - .mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-probe", - ssh_dest: "paperclip-probe.exe.xyz", - status: "running", - }), { status: 200 }), - ) - .mockResolvedValueOnce(new Response("{}", { status: 200 })); - queueSpawnResult({ code: 1, stderr: "permission denied" }); - - const result = await plugin.definition.onEnvironmentProbe?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - config: { - apiKey: "api-key", - }, - }); - - expect(result).toMatchObject({ - ok: false, - summary: "exe.dev environment probe failed.", - metadata: { - provider: "exe-dev", - }, - }); - expect(String(result?.metadata?.error ?? "")).toContain("permission denied"); - expect(String(fetchMock.mock.calls[1]?.[1]?.body ?? "")).toBe("rm --json 'paperclip-probe'"); - }); - - it("returns onboarding guidance when probe hits exe.dev SSH registration", async () => { - fetchMock - .mockResolvedValueOnce( - new Response(JSON.stringify({ - vm_name: "paperclip-probe", - ssh_dest: "paperclip-probe.exe.xyz", - status: "running", - }), { status: 200 }), - ) - .mockResolvedValueOnce(new Response("{}", { status: 200 })); - queueSpawnResult({ code: 1, stdout: "Please complete registration by running: ssh exe.dev\n" }); - - const result = await plugin.definition.onEnvironmentProbe?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - config: { - apiKey: "api-key", - }, - }); - - expect(result).toMatchObject({ - ok: false, - summary: "exe.dev environment probe failed.", - }); - expect(String(result?.metadata?.error ?? "")).toContain("the Paperclip host SSH key is not registered with exe.dev"); - expect(String(fetchMock.mock.calls[1]?.[1]?.body ?? "")).toBe("rm --json 'paperclip-probe'"); - }); - - it("deletes non-reusable leases on release", async () => { - fetchMock.mockResolvedValueOnce(new Response("{}", { status: 200 })); - - await plugin.definition.onEnvironmentReleaseLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - providerLeaseId: "vm-1", - config: { - apiKey: "api-key", - reuseLease: false, - }, - leaseMetadata: {}, - }); - - expect(String(fetchMock.mock.calls[0]?.[1]?.body ?? "")).toBe("rm --json 'vm-1'"); - }); - - it("destroys leases on demand", async () => { - fetchMock.mockResolvedValueOnce(new Response("{}", { status: 200 })); - - await plugin.definition.onEnvironmentDestroyLease?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - providerLeaseId: "vm-2", - config: { - apiKey: "api-key", - }, - leaseMetadata: {}, - }); - - expect(String(fetchMock.mock.calls[0]?.[1]?.body ?? "")).toBe("rm --json 'vm-2'"); - }); - - it("realizes a workspace by mkdir-ing the remote cwd over SSH when VM metadata is present", async () => { - queueSpawnResult({ code: 0, stdout: "", stderr: "" }); - - const result = await plugin.definition.onEnvironmentRealizeWorkspace?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - config: { - apiKey: "api-key", - timeoutMs: 300000, - }, - lease: { - providerLeaseId: "vm-1", - metadata: { - sshDest: "vm-1.exe.xyz", - remoteCwd: "/srv/paperclip/run-1", - }, - }, - workspace: { - localPath: "/local/paperclip", - remotePath: undefined, - }, - }); - - expect(spawnMock).toHaveBeenCalledTimes(1); - expect(spawnMock.mock.calls[0]?.[0]).toBe("ssh"); - const sshCommand = String(spawnMock.mock.calls[0]?.[1]?.at(-1) ?? ""); - expect(sshCommand).toContain("mkdir -p"); - expect(sshCommand).toContain("/srv/paperclip/run-1"); - expect(result).toMatchObject({ - cwd: "/srv/paperclip/run-1", - metadata: { - provider: "exe-dev", - remoteCwd: "/srv/paperclip/run-1", - }, - }); - }); - - it("falls back through workspace.remotePath then workspace.localPath when lease.metadata.remoteCwd is missing", async () => { - queueSpawnResult({ code: 0, stdout: "", stderr: "" }); - - const result = await plugin.definition.onEnvironmentRealizeWorkspace?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - config: { - apiKey: "api-key", - timeoutMs: 300000, - }, - lease: { - providerLeaseId: "vm-1", - metadata: { - sshDest: "vm-1.exe.xyz", - }, - }, - workspace: { - localPath: "/local/paperclip", - remotePath: "/srv/paperclip/remote-fallback", - }, - }); - - expect(result?.cwd).toBe("/srv/paperclip/remote-fallback"); - }); - - it("skips ensureRemoteWorkspace and returns the resolved cwd when no VM metadata is available", async () => { - const result = await plugin.definition.onEnvironmentRealizeWorkspace?.({ - driverKey: "exe-dev", - companyId: "company-1", - environmentId: "env-1", - config: { - apiKey: "api-key", - timeoutMs: 300000, - }, - lease: { - providerLeaseId: null, - metadata: { - remoteCwd: "/srv/paperclip/no-vm", - }, - }, - workspace: { - localPath: "/local/paperclip", - }, - }); - - expect(spawnMock).not.toHaveBeenCalled(); - expect(result?.cwd).toBe("/srv/paperclip/no-vm"); + it("bounds the cgroup, never the durable disk, by a lease deadline", async () => { + const expiresAt = new Date(Date.now() + 60000).toISOString(); + const lease = await hooks.onEnvironmentAcquireLease!({ ...params, runId: "a", requestedExpiresAt: expiresAt }); + expect(lease.expiresAt).toBe(expiresAt); + expect(transport.ssh.mock.calls.some(([, , cmd]) => cmd.includes("RuntimeMaxSec="))).toBe(true); }); }); -describe("exe-dev manifest form defaults", () => { - const configSchema = ( - manifest.environmentDrivers?.[0]?.configSchema as { - properties?: Record; - } - ); - const properties = configSchema.properties ?? {}; - - it("pre-fills VM sizing for the environment form", () => { - expect(properties.cpu?.default).toBe(4); - expect(properties.memory?.default).toBe("4GB"); - expect(properties.disk?.default).toBe("20GB"); +describe("configuration and capability contract", () => { + it.each([ + { mode: "attach", vmName: "-oProxyCommand=evil" }, + { mode: "create", image: "ubuntu:latest" }, + { strictHostKeyChecking: "no" }, + { setupScript: "curl install | sh" }, + { timeoutMs: NaN }, + ])("rejects unsafe or incompatible configuration %j", (patch) => { + expect(() => parseConfig({ ...config, ...patch })).toThrow(); }); - - it("declares no default on secret-ref fields, which would be persisted as a company secret", () => { - for (const prop of Object.values(properties)) { - if (prop.format === "secret-ref") { - expect(prop.default).toBeUndefined(); - } - } + it("requires explicit migration of old per-run environments", () => { + expect(() => parseConfig({ apiKey: "old-api-key", sshIdentityFile: "/key" })).toThrow("migrated explicitly"); + }); + it("advertises only implemented capabilities", () => { + const capabilities = manifest.environmentDrivers![0].sandboxCapabilities!; + expect(capabilities.duplexCommandStream).toBe(true); + expect(capabilities.runnerWebSocketIngress).toBe(false); + expect(capabilities.nativeSyncIn).toBe(false); + expect(capabilities.nativeSyncOut).toBe(false); + }); + it("rejects pasted public keys and truncated private keys", () => { + expect(validateSshPrivateKey("ssh-ed25519 AAAA public")).toContain("PUBLIC"); + expect(validateSshPrivateKey("-----BEGIN OPENSSH PRIVATE KEY-----\nAAAA")).toContain("footer"); }); }); diff --git a/packages/plugins/sandbox-providers/exe-dev/src/plugin.ts b/packages/plugins/sandbox-providers/exe-dev/src/plugin.ts index a586b665f4a..dd052a71103 100644 --- a/packages/plugins/sandbox-providers/exe-dev/src/plugin.ts +++ b/packages/plugins/sandbox-providers/exe-dev/src/plugin.ts @@ -1,966 +1,287 @@ -import path from "node:path"; -import { randomUUID } from "node:crypto"; -import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { spawn } from "node:child_process"; -import { definePlugin } from "@paperclipai/plugin-sdk"; +import { StringDecoder } from "node:string_decoder"; +import { createHash, randomUUID } from "node:crypto"; +import { createInterface } from "node:readline"; +import { definePlugin, decodeChannelBytes } from "@paperclipai/plugin-sdk"; import type { - PluginEnvironmentAcquireLeaseParams, - PluginEnvironmentDestroyLeaseParams, - PluginEnvironmentExecuteParams, - PluginEnvironmentExecuteResult, - PluginEnvironmentLease, - PluginEnvironmentProbeParams, - PluginEnvironmentProbeResult, - PluginEnvironmentRealizeWorkspaceParams, - PluginEnvironmentRealizeWorkspaceResult, - PluginEnvironmentReleaseLeaseParams, - PluginEnvironmentResumeLeaseParams, - PluginEnvironmentValidateConfigParams, - PluginEnvironmentValidationResult, + PluginContext, PluginEnvironmentAcquireLeaseParams, PluginEnvironmentDriverBaseParams, + PluginEnvironmentLease, PluginEnvironmentResumeLeaseParams, PluginEnvironmentExecuteResult, + PluginEnvironmentReleaseLeaseParams, PluginEnvironmentTerminationReceipt, } from "@paperclipai/plugin-sdk"; +import { openSsh, ssh, quote, type SshConfig } from "./transport.js"; +import { validateSshPrivateKey } from "./ssh-key.js"; +export { validateSshPrivateKey } from "./ssh-key.js"; -interface ExeDevDriverConfig { - apiKey: string | null; - apiUrl: string; - namePrefix: string; - image: string | null; - command: string | null; - cpu: number | null; - memory: string | null; - disk: string | null; - comment: string | null; - env: Record; - integrations: string[]; - tags: string[]; - setupScript: string | null; - prompt: string | null; - timeoutMs: number; - reuseLease: boolean; - sshUser: string | null; - sshPrivateKey: string | null; - sshIdentityFile: string | null; - sshPort: number; - strictHostKeyChecking: string; +const SUPERVISOR = "/opt/paperclip-exe/lease.mjs"; +const hash = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 32); +interface Config extends SshConfig { + vmName?: string; + mode: "create" | "attach"; + image?: string; + registryAuth?: string; + cpu: number; + memory: string; + disk: string; } - -interface ExeDevVmRecord { - name: string; - sshDest: string; - httpsUrl: string | null; - status: string | null; - region: string | null; - regionDisplay: string | null; -} - -interface SshExecutionResult { - exitCode: number | null; - signal: string | null; - timedOut: boolean; - stdout: string; - stderr: string; -} - -const DEFAULT_API_URL = "https://exe.dev/exec"; -const DEFAULT_TIMEOUT_MS = 300_000; -const EXE_DEV_API_MAX_TIMEOUT_MS = 29_000; -const SSH_SIGKILL_GRACE_MS = 250; -const MAX_VM_RECORD_DEPTH = 4; -const EXE_DEV_SSH_ONBOARDING_MARKER = "Please complete registration by running: ssh exe.dev"; -const EXE_DEV_SSH_EMAIL_PROMPT = "Please enter your email address:"; -const EXE_DEV_SSH_INVALID_KEY_FORMAT = /Load key [^\n]*invalid format/i; -const UUID_SECRET_REF_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; - -// exe.dev's `--setup-script` runs at VM init as the unprivileged `exedev` user, which -// has passwordless sudo. The Paperclip sandbox callback bridge is a Node script, so -// every Paperclip workload on this provider needs node on PATH before the bridge can -// start. When the operator hasn't supplied their own setup script, install Node 24 via -// nodesource so the VM comes up ready for Paperclip out of the box. -const DEFAULT_SETUP_SCRIPT = - "(command -v node >/dev/null 2>&1 && " + - "node -e 'const v=process.versions.node.split(\".\").map(Number);" + - "process.exit(v[0]>24||(v[0]===24&&v[1]>=11)?0:1)' >/dev/null 2>&1) || " + - "(curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - && " + - "sudo apt-get install -y nodejs)"; - -class ExeDevApiError extends Error { - readonly status: number; - readonly body: string; - - constructor(message: string, status: number, body: string) { - super(message); - this.name = "ExeDevApiError"; - this.status = status; - this.body = body; - } -} - -function parseOptionalString(value: unknown): string | null { - if (typeof value === "number" && Number.isFinite(value)) return String(value); - return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; -} - -function parseOptionalInteger(value: unknown): number | null { - if (value == null || value === "") return null; - const parsed = Number(value); - return Number.isFinite(parsed) ? Math.trunc(parsed) : null; -} - -function parseStringArray(value: unknown): string[] { - if (Array.isArray(value)) { - return value - .map((entry) => parseOptionalString(entry)) - .filter((entry): entry is string => entry != null); - } - if (typeof value === "string") { - return value - .split(",") - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0); - } - return []; -} - -function parseEnvMap(value: unknown): Record { - if (!value || typeof value !== "object" || Array.isArray(value)) return {}; - const env: Record = {}; - for (const [key, raw] of Object.entries(value)) { - const normalizedKey = key.trim(); - const normalizedValue = parseOptionalString(raw); - if (normalizedKey.length > 0 && normalizedValue != null) { - env[normalizedKey] = normalizedValue; - } - } - return env; -} - -function isValidUrl(value: string): boolean { - try { - new URL(value); - return true; - } catch { - return false; - } -} - -function isSecretRef(value: string): boolean { - return UUID_SECRET_REF_RE.test(value); -} - -// Catch the SSH-key paste failure modes we've seen in the wild (wrong file, -// PPK export, truncated paste) before the user pays the cost of provisioning a -// VM and getting a cryptic SSH error. Inline parse — no `ssh-keygen` dependency -// — so this also works on hosts where openssh-client isn't installed. -export function validateSshPrivateKey(rawKey: string): string | null { - const trimmed = rawKey.trim(); - if (!trimmed) return null; - - if (/^PuTTY-User-Key-File-\d/m.test(trimmed)) { - return "sshPrivateKey looks like a PuTTY .ppk file. Convert it to OpenSSH format (PuTTYgen → Conversions → Export OpenSSH key) and paste the resulting PEM."; - } - - if ( - /^(?:ssh-(?:rsa|dss|ed25519)|ecdsa-sha2-[a-z0-9-]+|sk-(?:ssh-ed25519|ecdsa-sha2-[a-z0-9-]+)@openssh\.com)\s+\S/.test( - trimmed, - ) - ) { - return "sshPrivateKey looks like a PUBLIC key. Paste the matching private key (the file without the .pub extension)."; - } - - const headerMatch = trimmed.match(/^-----BEGIN ([A-Z0-9 ]*)PRIVATE KEY-----/m); - if (!headerMatch) { - return "sshPrivateKey must be a PEM-encoded private key starting with a line like '-----BEGIN OPENSSH PRIVATE KEY-----'."; - } - - const footerMatch = trimmed.match(/^-----END ([A-Z0-9 ]*)PRIVATE KEY-----\s*$/m); - if (!footerMatch) { - return "sshPrivateKey is missing its '-----END … PRIVATE KEY-----' footer. Make sure you copied the whole file, including the final line."; - } - - const headerLabel = headerMatch[1].trim(); - const footerLabel = footerMatch[1].trim(); - if (headerLabel !== footerLabel) { - return `sshPrivateKey header/footer mismatch (BEGIN ${headerLabel || "(none)"} vs END ${footerLabel || "(none)"}). The file is likely truncated or two keys are concatenated.`; - } - - const headerLineEnd = trimmed.indexOf("\n", headerMatch.index ?? 0); - const footerStart = trimmed.lastIndexOf(footerMatch[0]); - if (headerLineEnd < 0 || footerStart <= headerLineEnd) { - return "sshPrivateKey appears to be empty between its BEGIN and END markers."; - } - - const bodyLines = trimmed - .slice(headerLineEnd + 1, footerStart) - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line.length > 0); - if (bodyLines.length === 0) { - return "sshPrivateKey appears to be empty between its BEGIN and END markers."; - } - - // PEM bodies are base64 lines, optionally preceded by `Header: value` lines - // on encrypted PKCS#1 keys (`Proc-Type:`, `DEK-Info:`). - const base64Line = /^[A-Za-z0-9+/=]+$/; - const pemHeaderLine = /^[A-Za-z][A-Za-z0-9-]*:\s.+$/; - for (const line of bodyLines) { - if (!base64Line.test(line) && !pemHeaderLine.test(line)) { - return "sshPrivateKey body contains non-base64 characters. The key may have been corrupted by line-wrapping or copy-paste."; - } - } - - return null; -} - -function normalizeApiUrl(value: string | null): string { - if (!value) return DEFAULT_API_URL; - const trimmed = value.trim(); - if (!trimmed) return DEFAULT_API_URL; - try { - const parsed = new URL(trimmed); - const normalizedPath = parsed.pathname.replace(/\/+$/, "") || "/"; - if (normalizedPath === "/exec") { - parsed.pathname = "/exec"; - return parsed.toString(); - } - parsed.pathname = `${normalizedPath === "/" ? "" : normalizedPath}/exec`.replace(/\/{2,}/g, "/"); - return parsed.toString(); - } catch { - return trimmed; - } -} - -function normalizeNamePrefix(value: string | null): string { - const normalized = (value ?? "paperclip") - .toLowerCase() - .replace(/[^a-z0-9-]+/g, "-") - .replace(/^-+|-+$/g, "") - .replace(/-{2,}/g, "-"); - return normalized.length > 0 ? normalized.slice(0, 24) : "paperclip"; -} - -function parseDriverConfig(raw: Record): ExeDevDriverConfig { - const timeoutMs = Number(raw.timeoutMs ?? DEFAULT_TIMEOUT_MS); - const sshPort = Number(raw.sshPort ?? 22); - - return { - apiKey: parseOptionalString(raw.apiKey), - apiUrl: normalizeApiUrl(parseOptionalString(raw.apiUrl)), - namePrefix: normalizeNamePrefix(parseOptionalString(raw.namePrefix)), - image: parseOptionalString(raw.image), - command: parseOptionalString(raw.command), - cpu: parseOptionalInteger(raw.cpu), - memory: parseOptionalString(raw.memory), - disk: parseOptionalString(raw.disk), - comment: parseOptionalString(raw.comment), - env: parseEnvMap(raw.env), - integrations: parseStringArray(raw.integrations), - tags: parseStringArray(raw.tags), - setupScript: parseOptionalString(raw.setupScript), - prompt: parseOptionalString(raw.prompt), - timeoutMs: Number.isFinite(timeoutMs) ? Math.trunc(timeoutMs) : DEFAULT_TIMEOUT_MS, - reuseLease: raw.reuseLease === true, - sshUser: parseOptionalString(raw.sshUser), - sshPrivateKey: parseOptionalString(raw.sshPrivateKey), - sshIdentityFile: parseOptionalString(raw.sshIdentityFile), - sshPort: Number.isFinite(sshPort) ? Math.trunc(sshPort) : 22, - strictHostKeyChecking: parseOptionalString(raw.strictHostKeyChecking) ?? "accept-new", +export function parseConfig(raw: Record): Config { + const str = (key: string) => typeof raw[key] === "string" && String(raw[key]).trim() ? String(raw[key]).trim() : undefined; + const mode = raw.mode ?? "attach"; + const config: Config = { + mode: mode as Config["mode"], vmName: str("vmName"), image: str("image"), + registryAuth: str("registryAuth") ?? process.env.EXE_DEV_REGISTRY_AUTH, + sshPrivateKey: str("sshPrivateKey") ?? process.env.EXE_DEV_SSH_PRIVATE_KEY, + sshIdentityFile: str("sshIdentityFile") ?? process.env.EXE_DEV_SSH_KEY_FILE, + knownHosts: str("knownHosts"), strictHostKeyChecking: (str("strictHostKeyChecking") ?? "accept-new") as Config["strictHostKeyChecking"], + timeoutMs: Number(raw.timeoutMs ?? 300000), cpu: Number(raw.cpu ?? 2), + memory: str("memory") ?? "4GB", disk: str("disk") ?? "20GB", }; -} - -function resolveApiKey(config: ExeDevDriverConfig): string { - if (config.apiKey) return config.apiKey; - const envApiKey = process.env.EXE_API_KEY?.trim() ?? ""; - if (!envApiKey) { - throw new Error("exe.dev environments require an API key in config or EXE_API_KEY."); + if (mode !== "create" && mode !== "attach") throw new Error("mode must be create or attach"); + if (config.vmName && !/^[a-z0-9][a-z0-9-]{0,62}$/.test(config.vmName)) throw new Error("Invalid VM name"); + if (mode === "attach" && !config.vmName) throw new Error("Attaching requires a named compatible VM; existing per-run configurations must be migrated explicitly"); + if (mode === "create" && !config.image?.match(/^[-a-zA-Z0-9./_:]+@sha256:[a-f0-9]{64}$/)) throw new Error("Creating requires a compatible image pinned by sha256 digest"); + if (!["yes", "accept-new"].includes(config.strictHostKeyChecking)) throw new Error("SSH host key checking cannot be disabled"); + if (!config.sshPrivateKey && !config.sshIdentityFile) throw new Error("Register an SSH key with exe.dev and configure sshPrivateKey or sshIdentityFile"); + if (config.sshPrivateKey && !/^[a-f0-9-]{36}$/i.test(config.sshPrivateKey)) { + const error = validateSshPrivateKey(config.sshPrivateKey); if (error) throw new Error(error); } - return envApiKey; -} - -function isValidShellEnvKey(value: string): boolean { - return /^[A-Za-z_][A-Za-z0-9_]*$/.test(value); -} - -function shellQuote(value: string): string { - return `'${value.replace(/'/g, `'"'"'`)}'`; -} - -function formatErrorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -function buildVmName(config: ExeDevDriverConfig, params: PluginEnvironmentAcquireLeaseParams): string { - const envPart = params.environmentId.replace(/[^a-z0-9]+/gi, "").slice(0, 8).toLowerCase() || "env"; - const runPart = params.runId.replace(/[^a-z0-9]+/gi, "").slice(0, 8).toLowerCase() || randomUUID().slice(0, 8); - return `${config.namePrefix}-${envPart}-${runPart}`.slice(0, 63); -} - -function buildFlag(name: string, value: string | number | null | undefined): string[] { - if (value == null) return []; - return [`--${name}=${shellQuote(String(value))}`]; -} - -function buildRepeatedFlag(name: string, values: string[]): string[] { - return values.flatMap((value) => buildFlag(name, value)); -} - -function buildEnvFlags(env: Record): string[] { - return Object.entries(env).flatMap(([key, value]) => buildFlag("env", `${key}=${value}`)); -} - -function resolveSetupScript(config: ExeDevDriverConfig): string | null { - if (config.setupScript === null) return DEFAULT_SETUP_SCRIPT; - const trimmed = config.setupScript.trim(); - return trimmed.length > 0 ? config.setupScript : null; -} - -function buildCreateCommand( - config: ExeDevDriverConfig, - vmName: string, -): string { - return [ - "new", - "--json", - "--no-email", - ...buildFlag("name", vmName), - ...buildFlag("image", config.image), - ...buildFlag("command", config.command), - ...buildFlag("cpu", config.cpu), - ...buildFlag("memory", config.memory), - ...buildFlag("disk", config.disk), - ...buildFlag("comment", config.comment), - ...buildEnvFlags(config.env), - ...buildRepeatedFlag("integration", config.integrations), - ...buildRepeatedFlag("tag", config.tags), - ...buildFlag("setup-script", resolveSetupScript(config)), - ...buildFlag("prompt", config.prompt), - ].join(" "); -} - -function replaceLiteralAll(input: string, search: string, replacement: string): string { - return search.length === 0 ? input : input.split(search).join(replacement); -} - -function redactCreateCommand(command: string, config: ExeDevDriverConfig): string { - let redacted = command; - - for (const [key, value] of Object.entries(config.env)) { - redacted = replaceLiteralAll( - redacted, - `--env=${shellQuote(`${key}=${value}`)}`, - `--env=${shellQuote(`${key}=[REDACTED]`)}`, - ); + if (!Number.isInteger(config.timeoutMs) || config.timeoutMs < 1000 || config.timeoutMs > 86400000) throw new Error("timeoutMs must be 1000–86400000"); + if (!Number.isInteger(config.cpu) || config.cpu < 1 || config.cpu > 64) throw new Error("Invalid CPU count"); + for (const size of [config.memory, config.disk]) if (!/^\d+(?:GB|G)?$/.test(size)) throw new Error("Invalid resource size"); + for (const obsolete of ["setupScript", "prompt", "command", "sshUser", "apiUrl"]) { + if (raw[obsolete]) throw new Error(`${obsolete} is not supported by durable exe.dev environments`); } - - if (config.prompt) { - redacted = replaceLiteralAll( - redacted, - `--prompt=${shellQuote(config.prompt)}`, - `--prompt=${shellQuote("[REDACTED]")}`, - ); - } - - const resolvedSetupScript = resolveSetupScript(config); - if (resolvedSetupScript && resolvedSetupScript !== DEFAULT_SETUP_SCRIPT) { - redacted = replaceLiteralAll( - redacted, - `--setup-script=${shellQuote(resolvedSetupScript)}`, - `--setup-script=${shellQuote("[REDACTED]")}`, - ); - } - - return redacted; + return config; } - -async function runLifecycleCommand( - config: ExeDevDriverConfig, - command: string, - logCommand = command, -): Promise { - const response = await fetch(config.apiUrl, { - method: "POST", - headers: { - Authorization: `Bearer ${resolveApiKey(config)}`, - "Content-Type": "text/plain; charset=utf-8", - }, - body: command, - signal: AbortSignal.timeout(Math.min(config.timeoutMs, EXE_DEV_API_MAX_TIMEOUT_MS)), - }); - const body = await response.text(); - if (!response.ok) { - throw new ExeDevApiError( - `exe.dev API command failed (${response.status}) for: ${logCommand}`, - response.status, - body, - ); - } - - const trimmed = body.trim(); - if (!trimmed) return null; - try { - return JSON.parse(trimmed); - } catch { - return body; - } +interface Scope { companyId: string; environmentId: string; } +interface Binding { + config: Config; scope: string; companyId: string; vm: string; root: string; unit: string; + leaseId: string; bindingId: string; image: Record; } +const bindings = new Map(); +const scopeId = (p: Scope) => hash(`${p.companyId}\0${p.environmentId}`); +const scopedKey = (p: Scope, leaseId: string) => `${scopeId(p)}:${leaseId}`; +const vmHost = (vm: string) => `${vm}.exe.xyz`; +let context: PluginContext | undefined; -function parseVmRecord(value: unknown, depth = 0): ExeDevVmRecord | null { - if (depth > MAX_VM_RECORD_DEPTH) return null; - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const record = value as Record; - const nested = parseVmRecord(record.vm, depth + 1) ?? parseVmRecord(record.data, depth + 1); - if (nested) return nested; - - const name = parseOptionalString(record.vm_name ?? record.name ?? record.vmName); - const sshDest = parseOptionalString(record.ssh_dest ?? record.sshDest) - ?? (name ? `${name}.exe.xyz` : null); - - if (!name || !sshDest) return null; - - return { - name, - sshDest, - httpsUrl: parseOptionalString(record.https_url ?? record.httpsUrl), - status: parseOptionalString(record.status), - region: parseOptionalString(record.region), - regionDisplay: parseOptionalString(record.region_display ?? record.regionDisplay), - }; +async function management(config: Config, command: string): Promise> { + let output: string; + try { output = await ssh(config, "exe.dev", command); } + catch (error) { throw new Error(config.registryAuth ? String((error as Error).message).split(config.registryAuth).join("[REDACTED]") : (error as Error).message); } + const result = JSON.parse(output); + if (result.error) throw new Error(`exe.dev: ${config.registryAuth ? String(result.error).split(config.registryAuth).join("[REDACTED]") : String(result.error)}`); + return result; } - -async function lookupVm(config: ExeDevDriverConfig, vmName: string): Promise { - const response = await runLifecycleCommand(config, `ls --json ${shellQuote(vmName)}`); - const list = Array.isArray((response as { vms?: unknown[] } | null)?.vms) - ? (response as { vms: unknown[] }).vms - : Array.isArray(response) - ? response - : response - ? [response] - : []; - for (const candidate of list) { - const parsed = parseVmRecord(candidate); - if (parsed?.name === vmName || parsed?.sshDest === vmName) { - return parsed; +async function findVm(config: Config, name: string): Promise | undefined> { + const result = await management(config, "ls -l --json"); + return (result.vms as Record[]).find((vm) => vm.vm_name === name); +} +class MissingVmError extends Error {} +async function resolveVm(config: Config, params: Scope & { resourceBinding?: { resourceId: string; identity: string; companyId: string } }, provision: boolean): Promise { + const name = config.vmName ?? `paperclip-${scopeId(params).slice(0, 20)}`; + if (params.resourceBinding && (params.resourceBinding.resourceId !== name || params.resourceBinding.companyId !== params.companyId)) throw new Error("Durable resource binding does not match this environment"); + let vm = await findVm(config, name); + if (!vm && provision && config.mode === "create" && !params.resourceBinding) { + // Name is stable across retries and competing workers. An uncertain create + // is reconciled by lookup; never issue a randomly named second VM. + try { + await management(config, `new --json --no-email --tag=${quote("paperclip-" + scopeId(params))} --name=${quote(name)} --image=${quote(config.image!)} ${config.registryAuth ? `--registry-auth=${quote(config.registryAuth)}` : ""} --cpu=${config.cpu} --memory=${quote(config.memory)} --disk=${quote(config.disk)}`); + } catch (error) { + vm = await findVm(config, name); + if (!vm) throw error; } + vm ??= await findVm(config, name); } - return null; + if (!vm) throw new MissingVmError(`exe.dev VM ${name} is missing; no automatic replacement is performed`); + if (vm.proxy_share !== "private") throw new Error("exe.dev VM must use private HTTP sharing"); + return name; +} +async function inspect(config: Config, vm: string): Promise> { + const output = await ssh(config, vmHost(vm), "test -r /opt/paperclip-exe/lease.mjs && command -v node >/dev/null && command -v paperclip-runnerd >/dev/null && test -r /opt/paperclip-runner/provider-pack/provider-pack.json && sudo -n true && cat /opt/paperclip-exe/image.json"); + const image = JSON.parse(output); + if (image.schema !== 1 || !image.sourceRevision || !image.contentId) throw new Error("Incompatible exe.dev image; recreate explicitly from a published Paperclip image"); + return image; +} +async function claim(config: Config, vm: string, scope: string): Promise { + // Serialized across controller processes, not merely a worker-local mutex. + const script = `const fs=require('fs'); const p='/var/lib/paperclip-exe/binding.json'; fs.mkdirSync('/var/lib/paperclip-exe',{recursive:true}); let b; try { b=JSON.parse(fs.readFileSync(p,'utf8')); } catch(e) { if(e.code!=='ENOENT') throw e; b={scope:${JSON.stringify(scope)},id:require('crypto').randomUUID()}; fs.writeFileSync(p,JSON.stringify(b),{flag:'wx',mode:384}); } if(b.scope!==${JSON.stringify(scope)}) throw Error('VM belongs to another Paperclip company/environment'); process.stdout.write(b.id);`; + return (await ssh(config, vmHost(vm), `sudo -n flock /var/lock/paperclip-exe-claim.lock node -e ${quote(script)}`)).trim(); +} +function decodeBinding(params: PluginEnvironmentDriverBaseParams, lease: PluginEnvironmentLease): Binding { + const config = parseConfig(params.config); + const scope = scopeId(params); + const match = lease.providerLeaseId?.match(/^([a-z0-9][a-z0-9-]{0,62}):([a-f0-9]{32}):([a-f0-9]{32})$/); + if (!match || match[2] !== scope || (config.vmName && config.vmName !== match[1])) throw new Error("Lease does not belong to this exe.dev environment"); + const bindingId = lease.metadata?.bindingId; + if (typeof bindingId !== "string" || !/^[a-f0-9-]{36}$/.test(bindingId)) throw new Error("Missing durable VM identity"); + return { config, scope, companyId: params.companyId, vm: match[1], root: `/var/lib/paperclip-exe/${scope}/${match[3]}`, + unit: `paperclip-lease-${match[3]}.service`, leaseId: lease.providerLeaseId!, bindingId, + image: (lease.metadata?.exeImageProvenance ?? {}) as Record }; +} +async function assertIdentity(binding: Binding) { + const actual = JSON.parse(await ssh(binding.config, vmHost(binding.vm), "sudo -n cat /var/lib/paperclip-exe/binding.json")); + if (actual.scope !== binding.scope || actual.id !== binding.bindingId) throw new Error("Durable VM identity changed; explicit recovery is required"); +} +async function start(binding: Binding, expiresAt?: string | null) { + const remaining = expiresAt ? Math.floor((Date.parse(expiresAt) - Date.now()) / 1000) : null; + if (remaining != null && (!Number.isFinite(remaining) || remaining < 1)) throw new Error("Lease deadline has expired"); + const script = [ + `mkdir -p ${quote(binding.root + "/workspace")} ${quote(binding.root + "/home")}`, + `chown exedev:exedev ${quote(binding.root)} ${quote(binding.root + "/home")} ${quote(binding.root + "/workspace")}`, + `if systemctl is-active --quiet ${quote(binding.unit)}; then :; else rm -f ${quote(binding.root + "/control.sock")}; systemd-run --quiet --collect --unit=${quote(binding.unit)} --uid=exedev --property=KillMode=control-group --property=TimeoutStopSec=10 ${remaining == null ? "" : `--property=RuntimeMaxSec=${remaining}`} /usr/bin/env HOME=${quote(binding.root + "/home")} PATH=/opt/paperclip-runner/provider-pack/node_modules/.bin:/usr/local/bin:/usr/bin:/bin PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=/opt/paperclip-runner/provider-pack node ${SUPERVISOR} serve ${quote(binding.root)}; fi`, + `for i in $(seq 1 100); do test -S ${quote(binding.root + "/control.sock")} && exit 0; sleep .1; done; exit 1`, + ].join(" && "); + await ssh(binding.config, vmHost(binding.vm), `sudo -n flock ${quote("/var/lock/" + binding.unit + ".lock")} sh -c ${quote(script)}`); +} +function leaseOf(binding: Binding, resumed: boolean, expiresAt?: string | null): PluginEnvironmentLease { + return { providerLeaseId: binding.leaseId, expiresAt: expiresAt ?? null, metadata: { + provider: "exe-dev", vmName: binding.vm, bindingId: binding.bindingId, + resourceLifetime: "environment", environmentResourceBinding: { provider: "exe-dev", companyId: binding.companyId, resourceId: binding.vm, identity: binding.bindingId }, remoteCwd: binding.root + "/workspace", remoteHome: binding.root + "/home", + httpsUrl: `https://${vmHost(binding.vm)}`, shellCommand: "bash", reuseLease: true, + resumedLease: resumed, exeImageProvenance: binding.image, leaseDeadline: expiresAt ?? null, + } }; +} +async function acquire(params: PluginEnvironmentAcquireLeaseParams) { + const config = parseConfig(params.config); + const vm = await resolveVm(config, params, true); + const image = await inspect(config, vm); + const scope = scopeId(params); + const bindingId = await claim(config, vm, scope); + if (params.resourceBinding && bindingId !== params.resourceBinding.identity) throw new Error("Durable VM identity changed; explicit recovery is required"); + const id = randomUUID().replaceAll("-", ""); + const binding: Binding = { config, vm, image, scope, companyId: params.companyId, bindingId, + root: `/var/lib/paperclip-exe/${scope}/${id}`, unit: `paperclip-lease-${id}.service`, leaseId: `${vm}:${scope}:${id}` }; + await start(binding, params.requestedExpiresAt); + bindings.set(scopedKey(params, binding.leaseId), binding); + return leaseOf(binding, false, params.requestedExpiresAt); +} +async function resume(params: PluginEnvironmentResumeLeaseParams) { + const binding = decodeBinding(params, { providerLeaseId: params.providerLeaseId, metadata: params.leaseMetadata }); + await resolveVm(binding.config, params, false); + await assertIdentity(binding); + binding.image = await inspect(binding.config, binding.vm); + await start(binding, params.leaseMetadata?.leaseDeadline as string | undefined); + bindings.set(scopedKey(params, binding.leaseId), binding); + return leaseOf(binding, true, params.leaseMetadata?.leaseDeadline as string | undefined); +} +async function stop(params: PluginEnvironmentReleaseLeaseParams): Promise { + if (!params.providerLeaseId) return; + const binding = decodeBinding(params, { providerLeaseId: params.providerLeaseId, metadata: params.leaseMetadata }); + await assertIdentity(binding); + // systemctl stop waits for the cgroup to empty. A lost SSH response does not + // produce a receipt. Repeat stop safely after reconnection to confirm it. + const script = `load=$(systemctl show -p LoadState --value ${quote(binding.unit)}); if test "$load" != not-found; then systemctl stop ${quote(binding.unit)} || exit $?; fi; state=$(systemctl show -p ActiveState --value ${quote(binding.unit)}); test "$state" = inactive -o "$state" = failed`; + await ssh(binding.config, vmHost(binding.vm), `sudo -n flock ${quote("/var/lock/" + binding.unit + ".lock")} sh -c ${quote(script)}`); + bindings.delete(scopedKey(params, binding.leaseId)); + return { providerLeaseId: binding.leaseId, state: "stopped" }; } -async function createVm( - config: ExeDevDriverConfig, - params: PluginEnvironmentAcquireLeaseParams | PluginEnvironmentProbeParams, -): Promise { - const vmName = "runId" in params - ? buildVmName(config, params) - : `${config.namePrefix}-probe-${randomUUID().slice(0, 8)}`.slice(0, 63); - const command = buildCreateCommand(config, vmName); - const response = await runLifecycleCommand(config, command, redactCreateCommand(command, config)); - const created = parseVmRecord(response) ?? await lookupVm(config, vmName); - if (!created) { - throw new Error(`exe.dev did not return VM metadata for ${vmName}.`); - } - return created; -} - -async function deleteVm(config: ExeDevDriverConfig, vmName: string): Promise { - await runLifecycleCommand(config, `rm --json ${shellQuote(vmName)}`); -} - -function buildSshDestination(config: ExeDevDriverConfig, vm: ExeDevVmRecord): string { - return config.sshUser ? `${config.sshUser}@${vm.sshDest}` : vm.sshDest; -} - -function buildSshArgs( - config: ExeDevDriverConfig, - vm: ExeDevVmRecord, - remoteCommand: string, - sshIdentityFile: string | null, -): string[] { - const args = [ - "-T", - "-o", - "BatchMode=yes", - "-o", - `StrictHostKeyChecking=${config.strictHostKeyChecking}`, - "-o", - "ConnectTimeout=15", - "-p", - String(config.sshPort), - ]; - if (sshIdentityFile) { - args.push("-i", sshIdentityFile, "-o", "IdentitiesOnly=yes"); - } - args.push(buildSshDestination(config, vm), remoteCommand); - return args; -} - -async function prepareSshIdentity(config: ExeDevDriverConfig): Promise<{ - sshIdentityFile: string | null; - cleanup: () => Promise; -}> { - if (!config.sshPrivateKey) { - return { - sshIdentityFile: config.sshIdentityFile, - cleanup: async () => {}, - }; - } - - const tempDir = await mkdtemp(path.join(tmpdir(), "paperclip-exe-dev-ssh-")); - const sshIdentityFile = path.join(tempDir, "id_ed25519"); - const privateKey = config.sshPrivateKey.endsWith("\n") - ? config.sshPrivateKey - : `${config.sshPrivateKey}\n`; - - await writeFile(sshIdentityFile, privateKey, { mode: 0o600 }); - await chmod(sshIdentityFile, 0o600); - - return { - sshIdentityFile, - cleanup: async () => { - await rm(tempDir, { recursive: true, force: true }); - }, - }; -} - -function buildLoginShellScript(input: { - command: string; - args: string[]; - cwd?: string; - env?: Record; -}): string { - const env = input.env ?? {}; - for (const key of Object.keys(env)) { - if (!isValidShellEnvKey(key)) { - throw new Error(`Invalid exe.dev environment variable key: ${key}`); - } - } - const envArgs = Object.entries(env) - .filter((entry): entry is [string, string] => typeof entry[1] === "string") - .map(([key, value]) => `${key}=${shellQuote(value)}`); - const commandParts = [shellQuote(input.command), ...input.args.map(shellQuote)].join(" "); - const finalLine = envArgs.length > 0 - ? `exec env ${envArgs.join(" ")} ${commandParts}` - : `exec ${commandParts}`; - // Source the common login profiles before exec so the command runs with the - // interactive-shell PATH. The wrapper sources no `nvm.sh`; the sandbox image - // supplies node on the PATH. - const lines = [ - 'if [ -f /etc/profile ]; then . /etc/profile >/dev/null 2>&1 || true; fi', - 'if [ -f "$HOME/.profile" ]; then . "$HOME/.profile" >/dev/null 2>&1 || true; fi', - 'if [ -f "$HOME/.bash_profile" ]; then . "$HOME/.bash_profile" >/dev/null 2>&1 || true; elif [ -f "$HOME/.bashrc" ]; then . "$HOME/.bashrc" >/dev/null 2>&1 || true; fi', - 'if [ -f "$HOME/.zprofile" ]; then . "$HOME/.zprofile" >/dev/null 2>&1 || true; fi', - ]; - if (input.cwd) { - lines.push(`cd ${shellQuote(input.cwd)}`); - } - lines.push(finalLine); - return lines.join(" && "); -} - -function formatSshFailure( - action: string, - vmName: string, - result: Pick, -): string { - const combinedOutput = `${result.stderr}\n${result.stdout}`; - if ( - combinedOutput.includes(EXE_DEV_SSH_ONBOARDING_MARKER) - || combinedOutput.includes(EXE_DEV_SSH_EMAIL_PROMPT) - ) { - return [ - `Failed to ${action} exe.dev VM ${vmName}: the Paperclip host SSH key is not registered with exe.dev.`, - "Complete exe.dev's one-time SSH onboarding on this host by running `ssh exe.dev` and following the email verification prompt, then retry.", - ].join(" "); - } - - if (EXE_DEV_SSH_INVALID_KEY_FORMAT.test(combinedOutput)) { - return [ - `Failed to ${action} exe.dev VM ${vmName}: the configured SSH private key isn't an OpenSSH-format private key.`, - "Confirm the secret starts with `-----BEGIN … PRIVATE KEY-----` and isn't the `.pub` file or a PuTTY `.ppk` export.", - ].join(" "); - } - - return `Failed to ${action} exe.dev VM ${vmName}: ${result.stderr.trim() || result.stdout.trim() || "unknown error"}`; -} - -async function runSshCommand( - config: ExeDevDriverConfig, - vm: ExeDevVmRecord, - remoteCommand: string, - options: { stdin?: string; timeoutMs?: number } = {}, -): Promise { - const timeoutMs = options.timeoutMs ?? config.timeoutMs; - const identity = await prepareSshIdentity(config); - - try { - return await new Promise((resolve, reject) => { - const child = spawn("ssh", buildSshArgs(config, vm, remoteCommand, identity.sshIdentityFile), { - stdio: [options.stdin != null ? "pipe" : "ignore", "pipe", "pipe"], - }); - let stdout = ""; - let stderr = ""; - let timedOut = false; - let killTimer: NodeJS.Timeout | null = null; - const timer = timeoutMs > 0 - ? setTimeout(() => { - timedOut = true; - child.kill("SIGTERM"); - killTimer = setTimeout(() => { - child.kill("SIGKILL"); - }, SSH_SIGKILL_GRACE_MS); - }, timeoutMs) - : null; - - child.stdout?.on("data", (chunk) => { - stdout += String(chunk); - }); - child.stderr?.on("data", (chunk) => { - stderr += String(chunk); - }); - child.on("error", (error) => { - if (timer) clearTimeout(timer); - if (killTimer) clearTimeout(killTimer); - reject(error); - }); - child.on("close", (code, signal) => { - if (timer) clearTimeout(timer); - if (killTimer) clearTimeout(killTimer); - resolve({ - exitCode: timedOut ? null : code, - signal, - timedOut, - stdout, - stderr, - }); - }); - - if (options.stdin != null && child.stdin) { - child.stdin.write(options.stdin); +interface Command { command: string; args?: readonly string[]; cwd?: string; env?: Record; timeoutMs?: number; } +async function channel(binding: Binding, command: Command, onData: (stream: "stdout" | "stderr", data: Buffer) => void) { + const { child } = await openSsh(binding.config, vmHost(binding.vm), `node ${SUPERVISOR} call ${quote(binding.root)}`); + let settled = false; + let stderr = ""; + let resolve!: (result: PluginEnvironmentExecuteResult) => void; + let reject!: (error: Error) => void; + const result = new Promise((yes, no) => { resolve = yes; reject = no; }); + const lines = createInterface({ input: child.stdout }); + const timer = setTimeout(() => { child.kill("SIGKILL"); }, (command.timeoutMs ?? binding.config.timeoutMs) + 20000); + child.stderr.on("data", (data) => { stderr = (stderr + String(data)).slice(-8192); }); + lines.on("line", (line) => { + try { + const frame = JSON.parse(line); + if (frame.type === "stdout" || frame.type === "stderr") onData(frame.type, Buffer.from(frame.data, "base64")); + else if (frame.type === "exit") { + settled = true; clearTimeout(timer); + if (frame.error) reject(new Error(frame.error)); + else resolve({ exitCode: frame.code, signal: frame.signal, timedOut: frame.timedOut, stdout: "", stderr: "" }); child.stdin.end(); } - }); - } finally { - await identity.cleanup(); - } -} - -async function detectRemoteContext( - config: ExeDevDriverConfig, - vm: ExeDevVmRecord, -): Promise<{ homeDir: string; shellCommand: "bash" | "sh" }> { - const result = await runSshCommand( - config, - vm, - `sh -lc ${shellQuote( - 'home="${HOME:-}"; if [ -z "$home" ]; then home="$(pwd)"; fi; if command -v bash >/dev/null 2>&1; then shell=bash; else shell=sh; fi; printf "%s\\n%s\\n" "$home" "$shell"', - )}`, - ); - if (result.timedOut || result.exitCode !== 0) { - throw new Error(formatSshFailure("inspect", vm.name, result)); - } - - const [homeDirRaw, shellRaw] = result.stdout.split(/\r?\n/); - const homeDir = homeDirRaw?.trim() || "/tmp"; - return { - homeDir, - shellCommand: shellRaw?.trim() === "bash" ? "bash" : "sh", - }; -} - -async function ensureRemoteWorkspace( - config: ExeDevDriverConfig, - vm: ExeDevVmRecord, - remoteCwd: string, -): Promise { - const result = await runSshCommand( - config, - vm, - `sh -lc ${shellQuote(`mkdir -p ${shellQuote(remoteCwd)}`)}`, - ); - if (result.timedOut || result.exitCode !== 0) { - throw new Error(formatSshFailure("create workspace for", vm.name, result)); - } -} - -async function buildLease( - config: ExeDevDriverConfig, - vm: ExeDevVmRecord, - requestedCwd: string | undefined, - resumedLease: boolean, -): Promise { - const remote = await detectRemoteContext(config, vm); - const remoteCwd = requestedCwd?.trim() || path.posix.join(remote.homeDir, "paperclip-workspace"); - await ensureRemoteWorkspace(config, vm, remoteCwd); - - return { - providerLeaseId: vm.name, - metadata: { - provider: "exe-dev", - vmName: vm.name, - sshDest: vm.sshDest, - httpsUrl: vm.httpsUrl, - region: vm.region, - regionDisplay: vm.regionDisplay, - shellCommand: remote.shellCommand, - remoteCwd, - timeoutMs: config.timeoutMs, - reuseLease: config.reuseLease, - resumedLease, - }, - }; -} - -function metadataVmRecord(params: { - providerLeaseId: string | null; - leaseMetadata?: Record | null; -}): ExeDevVmRecord | null { - if (!params.providerLeaseId) return null; - const sshDest = parseOptionalString(params.leaseMetadata?.sshDest) ?? `${params.providerLeaseId}.exe.xyz`; - return { - name: params.providerLeaseId, - sshDest, - httpsUrl: parseOptionalString(params.leaseMetadata?.httpsUrl), - status: parseOptionalString(params.leaseMetadata?.status), - region: parseOptionalString(params.leaseMetadata?.region), - regionDisplay: parseOptionalString(params.leaseMetadata?.regionDisplay), - }; + } catch { child.kill("SIGKILL"); } + }); + child.once("error", (error) => { clearTimeout(timer); reject(error); }); + child.once("close", () => { clearTimeout(timer); if (!settled) reject(new Error(`exe.dev command transport closed without an exit receipt: ${stderr}`)); }); + const send = (frame: unknown) => child.stdin.write(JSON.stringify(frame) + "\n"); + send({ type: "start", command: command.command, cwd: command.cwd, env: command.env, args: command.args ?? [], timeoutMs: command.timeoutMs ?? binding.config.timeoutMs }); + return { result, send, close: () => child.kill("SIGTERM") }; } +type Channel = Awaited>; +const channels = new Map(); const plugin = definePlugin({ - async setup(ctx) { - ctx.logger.info("exe.dev sandbox provider plugin ready"); + async setup(ctx) { context = ctx; }, + async onHealth() { return { status: "ok", message: "exe.dev durable environment provider ready" }; }, + async onEnvironmentValidateConfig(params) { + try { parseConfig(params.config); return { ok: true }; } + catch (error) { return { ok: false, errors: [String((error as Error).message)] }; } }, - - async onHealth() { - return { status: "ok", message: "exe.dev sandbox provider plugin healthy" }; + async onEnvironmentProbe(params) { + try { + const config = parseConfig(params.config); + const vm = await resolveVm(config, params, false); + const image = await inspect(config, vm); + return { ok: true, summary: `Compatible durable VM ${vm}; probe does not modify or delete it.`, metadata: { vmName: vm, image } }; + } catch (error) { + if (error instanceof MissingVmError && params.config.mode === "create") { + return { ok: true, summary: "SSH account verified. The pinned image will be checked when the first run creates the durable VM." }; + } + return { ok: false, summary: (error as Error).message }; + } }, - - async onEnvironmentValidateConfig( - params: PluginEnvironmentValidateConfigParams, - ): Promise { - const config = parseDriverConfig(params.config); - const errors: string[] = []; - const warnings: string[] = []; - - if (config.apiUrl && !isValidUrl(config.apiUrl)) { - errors.push("apiUrl must be a valid URL."); - } - if (config.timeoutMs < 1 || config.timeoutMs > 86_400_000) { - errors.push("timeoutMs must be between 1 and 86400000."); - } - if (config.cpu != null && config.cpu <= 0) { - errors.push("cpu must be greater than 0 when provided."); - } - if (config.sshPort < 1 || config.sshPort > 65_535) { - errors.push("sshPort must be between 1 and 65535."); - } - if (!config.apiKey && !(process.env.EXE_API_KEY?.trim())) { - errors.push("exe.dev environments require an API key in config or EXE_API_KEY."); - } - for (const key of Object.keys(config.env)) { - if (!isValidShellEnvKey(key)) { - errors.push(`env contains an invalid key: ${key}`); + onEnvironmentAcquireLease: acquire, + onEnvironmentResumeLease: resume, + async onEnvironmentReleaseLease(params) { if (params.cancelActiveWork || params.config.reuseLease !== true) return await stop(params); }, + onEnvironmentDestroyLease: stop, + async onEnvironmentRealizeWorkspace(params) { + const binding = decodeBinding(params, params.lease); + return { cwd: binding.root + "/workspace", metadata: { provider: "exe-dev", resourceLifetime: "environment" } }; + }, + async onEnvironmentExecute(params) { + const binding = decodeBinding(params, params.lease); + bindings.set(scopedKey(params, binding.leaseId), binding); + let stdout = ""; let stderr = ""; + const decoders = { stdout: new StringDecoder("utf8"), stderr: new StringDecoder("utf8") }; + const active = await channel(binding, params, (stream, data) => { + const text = decoders[stream].write(data); + if (stream === "stdout") stdout += text; else stderr += text; + context?.execution.log(stream, text); + }); + if (params.stdin) { + const bytes = Buffer.from(params.stdin); + for (let offset = 0; offset < bytes.length; offset += 65536) { + active.send({ type: "stdin", data: bytes.subarray(offset, offset + 65536).toString("base64") }); } } - if ( - typeof params.config.strictHostKeyChecking === "string" && - params.config.strictHostKeyChecking.trim().length === 0 - ) { - errors.push("strictHostKeyChecking cannot be empty."); - } - if (config.sshPrivateKey && !isSecretRef(config.sshPrivateKey)) { - const sshKeyError = validateSshPrivateKey(config.sshPrivateKey); - if (sshKeyError) errors.push(sshKeyError); - } - - warnings.push( - "The Paperclip host must have SSH access to the created exe.dev VM, and its SSH key must be registered with exe.dev. The API token only covers provisioning.", + active.send({ type: "end" }); + const result = await active.result; + return { ...result, stdout: stdout + decoders.stdout.end(), stderr: stderr + decoders.stderr.end() }; + }, + async onDuplexChannelOpen(params) { + if (channels.has(params.hostRouteId)) throw new Error("Duplicate duplex route"); + const binding = bindings.get(scopedKey(params, params.providerLeaseId)); + if (!binding || params.driverKey !== "exe-dev") throw new Error("Unknown scoped exe.dev lease"); + const id = randomUUID(); + const active = await channel(binding, { command: params.command[0], args: params.command.slice(1), timeoutMs: 86400000 }, (stream, data) => { + if (stream === "stdout") context?.duplexChannel.data(params.hostRouteId, id, data); + }); + channels.set(params.hostRouteId, { id, channel: active }); + void active.result.then( + (result) => context?.duplexChannel.exit(params.hostRouteId, id, result.exitCode), + () => context?.duplexChannel.exit(params.hostRouteId, id, null, true), ); - if (config.reuseLease) { - warnings.push("reuseLease keeps the VM alive between runs; this provider does not suspend retained VMs."); - } - - if (errors.length > 0) { - return { ok: false, errors, warnings }; - } - - return { - ok: true, - warnings, - normalizedConfig: { ...config }, - }; + return { hostRouteId: params.hostRouteId, workerSessionId: id }; }, - - async onEnvironmentProbe( - params: PluginEnvironmentProbeParams, - ): Promise { - const config = parseDriverConfig(params.config); - let vm: ExeDevVmRecord | null = null; - - try { - vm = await createVm(config, params); - const lease = await buildLease(config, vm, undefined, false); - return { - ok: true, - summary: `Connected to exe.dev VM ${vm.name}.`, - metadata: { - provider: "exe-dev", - vmName: vm.name, - sshDest: vm.sshDest, - timeoutMs: config.timeoutMs, - reuseLease: config.reuseLease, - remoteCwd: lease.metadata?.remoteCwd, - shellCommand: lease.metadata?.shellCommand, - }, - }; - } catch (error) { - return { - ok: false, - summary: "exe.dev environment probe failed.", - metadata: { - provider: "exe-dev", - timeoutMs: config.timeoutMs, - reuseLease: config.reuseLease, - error: formatErrorMessage(error), - }, - }; - } finally { - if (vm) { - await deleteVm(config, vm.name).catch(() => undefined); - } - } + async onDuplexChannelWrite(params) { + const entry = channels.get(params.hostRouteId); const data = decodeChannelBytes(params.data); + if (entry?.id === params.workerSessionId && data) entry.channel.send({ type: "stdin", data: Buffer.from(data).toString("base64") }); }, - - async onEnvironmentAcquireLease( - params: PluginEnvironmentAcquireLeaseParams, - ): Promise { - const config = parseDriverConfig(params.config); - const vm = await createVm(config, params); - try { - return await buildLease(config, vm, params.requestedCwd, false); - } catch (error) { - await deleteVm(config, vm.name).catch(() => undefined); - throw error; - } + async onDuplexChannelStop(params) { + const entry = channels.get(params.hostRouteId); + if (entry?.id === params.workerSessionId) entry.channel.send({ type: "cancel" }); }, - - async onEnvironmentResumeLease( - params: PluginEnvironmentResumeLeaseParams, - ): Promise { - const config = parseDriverConfig(params.config); - const vm = await lookupVm(config, params.providerLeaseId); - if (!vm) { - return { providerLeaseId: null, metadata: { expired: true } }; - } - const requestedCwd = parseOptionalString(params.leaseMetadata?.remoteCwd); - return await buildLease(config, vm, requestedCwd ?? undefined, true); - }, - - async onEnvironmentReleaseLease( - params: PluginEnvironmentReleaseLeaseParams, - ): Promise { - if (!params.providerLeaseId) return; - const config = parseDriverConfig(params.config); - if (config.reuseLease) return; - await deleteVm(config, params.providerLeaseId); - }, - - async onEnvironmentDestroyLease( - params: PluginEnvironmentDestroyLeaseParams, - ): Promise { - if (!params.providerLeaseId) return; - const config = parseDriverConfig(params.config); - await deleteVm(config, params.providerLeaseId); - }, - - async onEnvironmentRealizeWorkspace( - params: PluginEnvironmentRealizeWorkspaceParams, - ): Promise { - const config = parseDriverConfig(params.config); - const remoteCwd = - parseOptionalString(params.lease.metadata?.remoteCwd) - ?? params.workspace.remotePath - ?? params.workspace.localPath - ?? "/tmp/paperclip-workspace"; - - const vm = metadataVmRecord({ - providerLeaseId: params.lease.providerLeaseId, - leaseMetadata: params.lease.metadata, - }); - if (vm) { - await ensureRemoteWorkspace(config, vm, remoteCwd); - } - - return { - cwd: remoteCwd, - metadata: { - provider: "exe-dev", - remoteCwd, - }, - }; - }, - - async onEnvironmentExecute( - params: PluginEnvironmentExecuteParams, - ): Promise { - if (!params.lease.providerLeaseId) { - return { - exitCode: 1, - signal: null, - timedOut: false, - stdout: "", - stderr: "No provider lease ID available for execution.", - }; - } - - const config = parseDriverConfig(params.config); - const vm = metadataVmRecord({ - providerLeaseId: params.lease.providerLeaseId, - leaseMetadata: params.lease.metadata, - }); - if (!vm) { - return { - exitCode: 1, - signal: null, - timedOut: false, - stdout: "", - stderr: "No exe.dev VM metadata available for execution.", - }; - } - - const command = buildLoginShellScript({ - command: params.command, - args: params.args ?? [], - cwd: params.cwd ?? parseOptionalString(params.lease.metadata?.remoteCwd) ?? undefined, - env: params.env, - }); - // `buildLoginShellScript` already explicitly sources `/etc/profile`, - // `~/.profile`, `~/.bash_profile`/`~/.bashrc`, and `~/.zprofile`. Wrapping - // the result in `sh -lc` (login shell) would source the same files a - // second time, which can cause `PATH` duplication or unexpected behavior - // on VMs whose profile init isn't idempotent. Use `sh -c` here so the - // explicit sourcing inside the script is the single source of truth. - const result = await runSshCommand( - config, - vm, - `sh -c ${shellQuote(command)}`, - { stdin: params.stdin, timeoutMs: params.timeoutMs ?? config.timeoutMs }, - ); - - return { - exitCode: result.exitCode, - signal: result.signal, - timedOut: result.timedOut, - stdout: result.stdout, - stderr: - !result.timedOut && result.exitCode !== 0 - ? formatSshFailure("execute commands on", vm.name, result) - : result.stderr, - metadata: { - provider: "exe-dev", - vmName: vm.name, - sshDest: vm.sshDest, - }, - }; + async onDuplexChannelClose(params) { + const entry = channels.get(params.hostRouteId); + if (entry) { entry.channel.send({ type: "cancel" }); entry.channel.close(); channels.delete(params.hostRouteId); } + return { hostRouteId: params.hostRouteId, ...(entry ? { workerSessionId: entry.id } : {}) }; }, + async onShutdown() { for (const entry of channels.values()) entry.channel.close(); channels.clear(); bindings.clear(); }, }); - export default plugin; diff --git a/packages/plugins/sandbox-providers/exe-dev/src/ssh-key.ts b/packages/plugins/sandbox-providers/exe-dev/src/ssh-key.ts new file mode 100644 index 00000000000..989a48bfbde --- /dev/null +++ b/packages/plugins/sandbox-providers/exe-dev/src/ssh-key.ts @@ -0,0 +1,60 @@ +export function validateSshPrivateKey(rawKey: string): string | null { + const trimmed = rawKey.trim(); + if (!trimmed) return null; + + if (/^PuTTY-User-Key-File-\d/m.test(trimmed)) { + return "sshPrivateKey looks like a PuTTY .ppk file. Convert it to OpenSSH format (PuTTYgen → Conversions → Export OpenSSH key) and paste the resulting PEM."; + } + + if ( + /^(?:ssh-(?:rsa|dss|ed25519)|ecdsa-sha2-[a-z0-9-]+|sk-(?:ssh-ed25519|ecdsa-sha2-[a-z0-9-]+)@openssh\.com)\s+\S/.test( + trimmed, + ) + ) { + return "sshPrivateKey looks like a PUBLIC key. Paste the matching private key (the file without the .pub extension)."; + } + + const headerMatch = trimmed.match(/^-----BEGIN ([A-Z0-9 ]*)PRIVATE KEY-----/m); + if (!headerMatch) { + return "sshPrivateKey must be a PEM-encoded private key starting with a line like '-----BEGIN OPENSSH PRIVATE KEY-----'."; + } + + const footerMatch = trimmed.match(/^-----END ([A-Z0-9 ]*)PRIVATE KEY-----\s*$/m); + if (!footerMatch) { + return "sshPrivateKey is missing its '-----END … PRIVATE KEY-----' footer. Make sure you copied the whole file, including the final line."; + } + + const headerLabel = headerMatch[1].trim(); + const footerLabel = footerMatch[1].trim(); + if (headerLabel !== footerLabel) { + return `sshPrivateKey header/footer mismatch (BEGIN ${headerLabel || "(none)"} vs END ${footerLabel || "(none)"}). The file is likely truncated or two keys are concatenated.`; + } + + const headerLineEnd = trimmed.indexOf("\n", headerMatch.index ?? 0); + const footerStart = trimmed.lastIndexOf(footerMatch[0]); + if (headerLineEnd < 0 || footerStart <= headerLineEnd) { + return "sshPrivateKey appears to be empty between its BEGIN and END markers."; + } + + const bodyLines = trimmed + .slice(headerLineEnd + 1, footerStart) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + if (bodyLines.length === 0) { + return "sshPrivateKey appears to be empty between its BEGIN and END markers."; + } + + // PEM bodies are base64 lines, optionally preceded by `Header: value` lines + // on encrypted PKCS#1 keys (`Proc-Type:`, `DEK-Info:`). + const base64Line = /^[A-Za-z0-9+/=]+$/; + const pemHeaderLine = /^[A-Za-z][A-Za-z0-9-]*:\s.+$/; + for (const line of bodyLines) { + if (!base64Line.test(line) && !pemHeaderLine.test(line)) { + return "sshPrivateKey body contains non-base64 characters. The key may have been corrupted by line-wrapping or copy-paste."; + } + } + + return null; +} + diff --git a/packages/plugins/sandbox-providers/exe-dev/src/transport.ts b/packages/plugins/sandbox-providers/exe-dev/src/transport.ts new file mode 100644 index 00000000000..44445554e10 --- /dev/null +++ b/packages/plugins/sandbox-providers/exe-dev/src/transport.ts @@ -0,0 +1,70 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import path from "node:path"; + +export interface SshConfig { + sshPrivateKey?: string; + sshIdentityFile?: string; + knownHosts?: string; + strictHostKeyChecking: "yes" | "accept-new"; + timeoutMs: number; +} + +export const quote = (value: string): string => `'${value.replace(/'/g, `'"'"'`)}'`; + +export async function openSsh(config: SshConfig, host: string, command: string): Promise<{ + child: ChildProcessWithoutNullStreams; + cleanup: () => Promise; +}> { + if (host !== "exe.dev" && !/^[a-z0-9][a-z0-9-]{0,62}\.exe\.xyz$/.test(host)) throw new Error("Invalid exe.dev SSH host"); + const temporary = await mkdtemp(path.join(tmpdir(), "paperclip-exe-")); + try { + let identity = config.sshIdentityFile; + if (config.sshPrivateKey) { + identity = path.join(temporary, "identity"); + await writeFile(identity, config.sshPrivateKey.trim() + "\n", { mode: 0o600 }); + } + const hostDirectory = path.join(homedir(), ".ssh"); + await mkdir(hostDirectory, { recursive: true, mode: 0o700 }); + let knownHosts = path.join(hostDirectory, "paperclip-exe-known_hosts"); + if (config.knownHosts) { + knownHosts = path.join(temporary, "known_hosts"); + await writeFile(knownHosts, config.knownHosts + "\n", { mode: 0o600 }); + } + const args = ["-F", "/dev/null", "-T", "-o", "BatchMode=yes", "-o", "ForwardAgent=no", + "-o", "ClearAllForwardings=yes", "-o", "ConnectTimeout=15", "-o", "ServerAliveInterval=15", + "-o", "ServerAliveCountMax=3", "-o", `StrictHostKeyChecking=${config.knownHosts ? "yes" : config.strictHostKeyChecking}`, + "-o", `UserKnownHostsFile=${knownHosts}`]; + if (identity) args.push("-i", identity, "-o", "IdentitiesOnly=yes"); + args.push(host, command); + const child = spawn("ssh", args, { stdio: "pipe" }); + child.stdin.on("error", () => {}); + const cleanup = async () => { await rm(temporary, { recursive: true, force: true }); }; + child.once("close", () => { void cleanup(); }); + return { child, cleanup }; + } catch (error) { + await rm(temporary, { recursive: true, force: true }); + throw error; + } +} + +export async function ssh(config: SshConfig, host: string, command: string, stdin?: string): Promise { + const { child } = await openSsh(config, host, command); + return await new Promise((resolve, reject) => { + let stdout = ""; let stderr = ""; let expired = false; + const timer = setTimeout(() => { expired = true; child.kill("SIGKILL"); }, config.timeoutMs); + child.stdout.on("data", (chunk) => { + stdout += String(chunk); + if (stdout.length > 32 * 1024 * 1024) child.kill("SIGKILL"); + }); + child.stderr.on("data", (chunk) => { stderr = (stderr + String(chunk)).slice(-8192); }); + child.once("error", (error) => { clearTimeout(timer); reject(error); }); + child.once("close", (code) => { + clearTimeout(timer); + if (code !== 0 || expired) reject(new Error(expired ? "exe.dev SSH deadline exceeded; remote state requires reconciliation" : `exe.dev SSH failed (${code}): ${stderr}`)); + else resolve(stdout); + }); + child.stdin.end(stdin); + }); +} diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index 1cf5adf0dda..9e8c5e598b4 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -175,6 +175,7 @@ export type { RefreshExternalObjectsResult, PluginEnvironmentDiagnostic, PluginEnvironmentDriverBaseParams, + PluginEnvironmentResourceBinding, PluginEnvironmentValidateConfigParams, PluginEnvironmentValidationResult, PluginEnvironmentProbeParams, diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index a599a8fc7e8..994109ccf83 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -579,7 +579,16 @@ export interface PluginEnvironmentDiagnostic { details?: Record; } +export interface PluginEnvironmentResourceBinding { + provider: string; + companyId: string; + resourceId: string; + identity: string; +} + export interface PluginEnvironmentDriverBaseParams { + /** Host-owned durable resource identity, independent of run lease lifetime. */ + resourceBinding?: PluginEnvironmentResourceBinding; driverKey: string; companyId: string; environmentId: string; diff --git a/packages/shared/src/feature-catalog.ts b/packages/shared/src/feature-catalog.ts index 879661d9dda..39dfa20f5b0 100644 --- a/packages/shared/src/feature-catalog.ts +++ b/packages/shared/src/feature-catalog.ts @@ -50,6 +50,13 @@ export const INSTANCE_FEATURE_CATALOG: Record { const i = args.indexOf(name); return i < 0 ? undefined : args[i + 1]; }; +const dockerfile = await readFile(path.join(root, "docker/exe-dev-runner/Dockerfile"), "utf8"); +const revision = execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); +const dirty = execFileSync("git", ["status", "--porcelain"], { cwd: root, encoding: "utf8" }).trim().length > 0; +if (dirty && process.env.PAPERCLIP_IMAGE_ALLOW_DIRTY !== "1") throw new Error("Commit the image inputs before building, or explicitly set PAPERCLIP_IMAGE_ALLOW_DIRTY=1 for a development image"); +// COPY packages includes the workspace graph. Hash every source-owned package +// input so changes outside the runner can never reuse an old content tag. +const packageInputs = execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z", "packages"], { cwd: root, encoding: "utf8" }).split("\0").filter(Boolean); +const contentId = await computeDaytonaImageContentId({ + repositoryRoot: root, + inputPaths: [...new Set([...DAYTONA_IMAGE_INPUT_PATHS.filter((p) => p !== "docker/daytona-runner/Dockerfile" && !p.startsWith("packages/")), ...packageInputs, "server/package.json", "ui/package.json", "cli/package.json", "docker/exe-dev-runner", "scripts/build-exe-dev-image.ts"])].sort(), + baseImages: [...dockerfile.matchAll(/^FROM (\S+@sha256:[a-f0-9]{64})/gm)].map((match) => match[1]), + frontendDigest: extractDaytonaDockerfileFrontendDigest(dockerfile), +}); +if (args.includes("--content-id")) { console.log(contentId); process.exit(0); } +const tag = option("--tag") ?? `ghcr.io/paperclipai/paperclip-exe-runner:experimental-${contentId}`; +const build = spawnSync("docker", ["buildx", "build", "--platform", "linux/amd64", "--file", "docker/exe-dev-runner/Dockerfile", + "--build-arg", `PAPERCLIP_RUNNER_SOURCE_REVISION=${revision}`, + "--build-arg", `PAPERCLIP_RUNNER_CONTENT_ID=${contentId}`, + "--tag", tag, ...(args.includes("--push") ? ["--push"] : ["--load"]), + ...(option("--metadata-file") ? ["--metadata-file", option("--metadata-file")!] : []), "."], { cwd: root, stdio: "inherit" }); +if (build.error) throw build.error; +process.exitCode = build.status ?? 1; diff --git a/scripts/check-node-version-policy.mjs b/scripts/check-node-version-policy.mjs index cceb9c75196..67f8b958476 100644 --- a/scripts/check-node-version-policy.mjs +++ b/scripts/check-node-version-policy.mjs @@ -64,7 +64,7 @@ const requiredSourceFragments = [ ["scripts/install.sh", "DEFAULT_NODE_MAJOR=24"], ["docker/agent-runtime/Dockerfile.base", "ARG NODE_VERSION=24"], ["packages/adapter-utils/src/sandbox-install-command.ts", "NODE_VERSION=\"v24.11.0\""], - ["packages/plugins/sandbox-providers/exe-dev/src/plugin.ts", "nodesource.com/setup_24.x"], + ["docker/exe-dev-runner/Dockerfile", "FROM node:24-bookworm@sha256:"], ["packages/plugins/sandbox-providers/modal/src/manifest.ts", 'default: "node:24"'], ["cli/esbuild.config.mjs", 'target: "node24"'], ["packages/plugins/sdk/src/bundlers.ts", 'target: "node24"'], diff --git a/server/src/__tests__/exe-environment-resource.test.ts b/server/src/__tests__/exe-environment-resource.test.ts new file mode 100644 index 00000000000..26d455e48b1 --- /dev/null +++ b/server/src/__tests__/exe-environment-resource.test.ts @@ -0,0 +1,61 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createDb, environments } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { bindEnvironmentResource, readEnvironmentResourceBinding } from "../services/environment-resource-binding.js"; +import { assertExeEnvironmentEnabled } from "../services/exe-environment-gate.js"; +import { environmentService } from "../services/environments.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("durable exe.dev environment resource", () => { + let db: ReturnType; + let stop: () => Promise; + beforeAll(async () => { + const started = await startEmbeddedPostgresTestDatabase("exe-resource"); + db = createDb(started.connectionString); stop = started.stop; + }); + afterAll(async () => { await stop?.(); }); + async function fixture() { + const id = randomUUID(); const companyId = randomUUID(); + await db.insert(environments).values({ id, name: id, driver: "sandbox", config: { provider: "exe-dev" } }); + const binding = { provider: "exe-dev", companyId, resourceId: "test-vm", identity: randomUUID() }; + return { id, companyId, binding }; + } + it("defaults off, gates API/runtime admission, and allows explicit opt-in", async () => { + const environment = { driver: "sandbox", config: { provider: "exe-dev" } }; + await expect(assertExeEnvironmentEnabled(db, environment)).rejects.toThrow("Enable experimental"); + await expect(assertExeEnvironmentEnabled(db, { driver: "sandbox", config: { provider: "daytona" } })).resolves.toBeUndefined(); + await instanceSettingsService(db).updateExperimental({ enableExeEnvironments: true }); + await expect(assertExeEnvironmentEnabled(db, environment)).resolves.toBeUndefined(); + await instanceSettingsService(db).updateExperimental({ enableExeEnvironments: false }); + }); + it("atomically converges concurrent acquisitions on the same durable identity", async () => { + const { id, companyId, binding } = await fixture(); + await Promise.all(Array.from({ length: 8 }, () => bindEnvironmentResource(db, id, companyId, binding))); + expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding); + }); + it("allows only one identity when competing first acquisitions disagree", async () => { + const { id, companyId, binding } = await fixture(); + const results = await Promise.allSettled([ + bindEnvironmentResource(db, id, companyId, binding), + bindEnvironmentResource(db, id, companyId, { ...binding, identity: randomUUID() }), + ]); + expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); + expect(results.filter((result) => result.status === "rejected")).toHaveLength(1); + }); + it("survives metadata replacement and has no dependence on run lease rows", async () => { + const { id, companyId, binding } = await fixture(); + await bindEnvironmentResource(db, id, companyId, binding); + await environmentService(db).update(id, { metadata: { note: "operator edit" } }); + expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding); + await environmentService(db).update(id, { metadata: null }); + expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding); + }); + it("denies a different company and a replacement VM", async () => { + const { id, companyId, binding } = await fixture(); + await bindEnvironmentResource(db, id, companyId, binding); + await expect(readEnvironmentResourceBinding(db, id, randomUUID())).rejects.toThrow("another company"); + await expect(bindEnvironmentResource(db, id, companyId, { ...binding, identity: randomUUID() })).rejects.toThrow("identity changed"); + expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding); + }); +}); diff --git a/server/src/__tests__/redaction.test.ts b/server/src/__tests__/redaction.test.ts index a4c903ec77e..66955944353 100644 --- a/server/src/__tests__/redaction.test.ts +++ b/server/src/__tests__/redaction.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { PRP_V1_EVENT_TYPES, + PRP_V2_EVENT_TYPES, REDACTED_EVENT_VALUE, redactAgentAdapterConfig, redactEventPayload, @@ -10,6 +11,16 @@ import { } from "../redaction.js"; describe("redaction", () => { + it("preserves only the closed v2 event catalog in matching v2 envelopes", () => { + const schema = JSON.parse(readFileSync(new URL("../../../packages/paperclip-runner/protocol/schemas/event-v2.schema.json", import.meta.url), "utf8")); + expect([...PRP_V2_EVENT_TYPES].sort()).toEqual([...schema.properties.eventType.enum].sort()); + for (const eventType of PRP_V2_EVENT_TYPES) { + const clean = redactEventPayload({ schema: "paperclip.prp.event.v2", schemaVersion: 2, eventType, apiKey: "sensitive" }); + expect(clean).toMatchObject({ schema: "paperclip.prp.event.v2", schemaVersion: 2, eventType, apiKey: REDACTED_EVENT_VALUE }); + } + expect(redactEventPayload({ schema: "paperclip.prp.event.v2", schemaVersion: 2, eventType: "unknown.provider.credential" })?.eventType).toBe(REDACTED_EVENT_VALUE); + expect(redactEventPayload({ schema: "paperclip.prp.event.v1", schemaVersion: 1, eventType: "session.goal.updated" })?.eventType).toBe(REDACTED_EVENT_VALUE); + }); it("keeps the discriminator allowlist in exact PRP v1 schema parity", () => { const schema = JSON.parse( readFileSync( diff --git a/server/src/redaction.ts b/server/src/redaction.ts index ebbbff26af5..b946a38c494 100644 --- a/server/src/redaction.ts +++ b/server/src/redaction.ts @@ -110,6 +110,7 @@ export const PAPERCLIP_PUBLIC_SCHEMA_IDS = new Set([ "paperclip.prp.command.v1", "paperclip.prp.contract_manifest.v1", "paperclip.prp.event.v1", + "paperclip.prp.event.v2", "paperclip.prp.fixture.v1", "paperclip.prp.identity.v1", "paperclip.prp.semantic_tool.v1", @@ -303,6 +304,10 @@ export const PRP_V1_EVENT_TYPES = new Set([ "issue.status.decision.superseded", "run.terminal", ]); +export const PRP_V2_EVENT_TYPES = new Set([ + ...[...PRP_V1_EVENT_TYPES].filter((type) => type !== "runner.backpressure" && type !== "semantic_tool.reconciled"), + "session.capabilities.updated", "session.goal.snapshot", "session.goal.updated", "session.goal.cleared", +]); const NATIVE_RUN_SPAN_SCHEMA = "paperclip.run-performance-span.v1"; const NATIVE_RUN_SPAN_FIELDS = ["span", "parentSpan"] as const; const NATIVE_RUN_SPAN_NAMES = new Set([ @@ -843,10 +848,9 @@ function isKnownPrpEventDiscriminator( ): value is string { return ( key === "eventType" && - container.schema === "paperclip.prp.event.v1" && - container.schemaVersion === 1 && typeof value === "string" && - PRP_V1_EVENT_TYPES.has(value) + ((container.schema === "paperclip.prp.event.v1" && container.schemaVersion === 1 && PRP_V1_EVENT_TYPES.has(value)) || + (container.schema === "paperclip.prp.event.v2" && container.schemaVersion === 2 && PRP_V2_EVENT_TYPES.has(value))) ); } diff --git a/server/src/routes/environments.ts b/server/src/routes/environments.ts index 68d51234cb5..3dccd4b53fd 100644 --- a/server/src/routes/environments.ts +++ b/server/src/routes/environments.ts @@ -1,3 +1,4 @@ +import { assertExeEnvironmentEnabled } from "../services/exe-environment-gate.js"; import { Router, type Request } from "express"; import type { Db } from "@paperclipai/db"; import { @@ -311,6 +312,9 @@ async function assertPlatformProvisionedEnvironmentWritable( * Clearing (null/false) is allowed so stale markers can be removed. */ function assertNoClientPlatformProvisionedMarkers(metadata: unknown): void { + if (isPlainRecord(metadata) && "environmentResourceBinding" in metadata) { + throw unprocessable("metadata.environmentResourceBinding is managed by Paperclip"); + } if (!isCloudManagedInstance() || !isPlainRecord(metadata)) return; for (const key of PLATFORM_PROVISIONED_MARKER_KEYS) { if (metadata[key] !== undefined && metadata[key] !== null && metadata[key] !== false) { @@ -723,11 +727,12 @@ export function environmentRoutes( router.get("/companies/:companyId/environments/capabilities", async (req, res) => { assertCanReadInstanceEnvironments(req); - const pluginDrivers = await listReadyPluginEnvironmentDrivers({ + const exeEnabled = (await instanceSettings.getExperimental()).enableExeEnvironments; + const pluginDrivers = (await listReadyPluginEnvironmentDrivers({ db, workerManager: options.pluginWorkerManager, recoverMissingWorker: options.recoverMissingPluginWorker, - }); + })).filter((driver) => driver.driverKey !== "exe-dev" || exeEnabled); res.json(getEnvironmentCapabilities( AGENT_ADAPTER_TYPES, { @@ -1015,6 +1020,7 @@ export function environmentRoutes( const companyId = req.params.companyId as string; assertCanAccessInstanceEnvironments(req); assertNoClientPlatformProvisionedMarkers(req.body.metadata); + await assertExeEnvironmentEnabled(db, req.body); if (req.body.driver === "local") { const existingLocal = await svc.list({ driver: "local" }); if (existingLocal.length > 0) { @@ -1129,6 +1135,7 @@ export function environmentRoutes( res.status(404).json({ error: "Environment not found" }); return; } + await assertExeEnvironmentEnabled(db, { driver: req.body.driver ?? existing.driver, config: req.body.config ?? existing.config }); await assertPlatformProvisionedEnvironmentWritable(existing, { patchBody: req.body, isForcedKubernetesExecution: async () => diff --git a/server/src/services/environment-probe.ts b/server/src/services/environment-probe.ts index ee1bce5d4c4..283a29900c0 100644 --- a/server/src/services/environment-probe.ts +++ b/server/src/services/environment-probe.ts @@ -1,3 +1,4 @@ +import { assertExeEnvironmentEnabled } from "./exe-environment-gate.js"; import type { Environment, EnvironmentProbeResult } from "@paperclipai/shared"; import type { Db } from "@paperclipai/db"; import { ensureSshWorkspaceReady } from "@paperclipai/adapter-utils/ssh"; @@ -23,6 +24,7 @@ export async function probeEnvironment( acquireSandboxRuntimeLease?: boolean; } = {}, ): Promise { + await assertExeEnvironmentEnabled(db, environment); const resolvedCompanyId = options.companyId ?? null; const parsed = options.resolvedConfig ?? ( options.acquireSandboxRuntimeLease === true @@ -47,7 +49,7 @@ export async function probeEnvironment( } if (parsed.driver === "sandbox") { - if (options.acquireSandboxRuntimeLease) { + if (options.acquireSandboxRuntimeLease && parsed.config.provider !== "exe-dev") { if (!resolvedCompanyId) { return { ok: false, diff --git a/server/src/services/environment-resource-binding.ts b/server/src/services/environment-resource-binding.ts new file mode 100644 index 00000000000..3767691bf1f --- /dev/null +++ b/server/src/services/environment-resource-binding.ts @@ -0,0 +1,34 @@ +import { and, eq, sql } from "drizzle-orm"; +import { environments, type Db } from "@paperclipai/db"; +import type { PluginEnvironmentResourceBinding } from "@paperclipai/plugin-sdk"; +import { conflict } from "../errors.js"; + +export function parseResourceBinding(value: unknown): PluginEnvironmentResourceBinding | undefined { + if (value == null) return undefined; + const binding = value as PluginEnvironmentResourceBinding; + if (binding.provider !== "exe-dev" || typeof binding.companyId !== "string" || !binding.companyId || typeof binding.resourceId !== "string" || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(binding.resourceId) || typeof binding.identity !== "string" || !/^[a-f0-9-]{36}$/.test(binding.identity)) { + throw conflict("Invalid durable environment resource binding"); + } + return { provider: binding.provider, companyId: binding.companyId, resourceId: binding.resourceId, identity: binding.identity }; +} + +export async function readEnvironmentResourceBinding(db: Db, environmentId: string, companyId: string) { + // Environments are instance-scoped; the durable binding carries ownership. + // Lease insertion separately rechecks the existing company binding contract. + const [row] = await db.select({ metadata: environments.metadata }).from(environments).where(eq(environments.id, environmentId)); + const binding = parseResourceBinding(row?.metadata?.environmentResourceBinding); + if (binding && binding.companyId !== companyId) throw conflict("Durable VM belongs to another company"); + return binding; +} + +/** Atomic compare-and-set: concurrent first acquisitions must agree on identity. */ +export async function bindEnvironmentResource(db: Db, environmentId: string, companyId: string, value: unknown) { + const binding = parseResourceBinding(value); + if (!binding || binding.companyId !== companyId) throw conflict("Provider did not attest a company-scoped durable VM binding"); + const encoded = JSON.stringify(binding); + const rows = await db.update(environments).set({ + metadata: sql`coalesce(${environments.metadata}, '{}'::jsonb) || jsonb_build_object('environmentResourceBinding', ${encoded}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(environments.id, environmentId), sql`(${environments.metadata}->'environmentResourceBinding' IS NULL OR ${environments.metadata}->'environmentResourceBinding' = ${encoded}::jsonb)`)).returning({ id: environments.id }); + if (rows.length !== 1) throw conflict("Durable VM identity changed; explicit recovery to a new environment is required"); +} diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 2deb410986e..9dd2ccc3373 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,3 +1,5 @@ +import { readEnvironmentResourceBinding, bindEnvironmentResource } from "./environment-resource-binding.js"; +import { assertExeEnvironmentEnabled } from "./exe-environment-gate.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; import { createHash, randomUUID } from "node:crypto"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -1100,6 +1102,7 @@ function createLocalEnvironmentDriver(db: Db): EnvironmentRuntimeDriver { driver: "local", async acquireRunLease(input) { + await assertExeEnvironmentEnabled(db, input.environment); return await environmentsSvc.acquireLease({ companyId: input.companyId, environmentId: input.environment.id, @@ -1154,6 +1157,7 @@ function createSshEnvironmentDriver(db: Db): EnvironmentRuntimeDriver { driver: "ssh", async acquireRunLease(input) { + await assertExeEnvironmentEnabled(db, input.environment); const parsed = await resolveEnvironmentDriverConfigForRuntime(db, input.companyId, input.environment, { issueId: input.issueId, heartbeatRunId: input.heartbeatRunId, @@ -1752,6 +1756,7 @@ function createSandboxEnvironmentDriver( driver: "sandbox", async acquireRunLease(input) { + await assertExeEnvironmentEnabled(db, input.environment); const storedParsed = parseEnvironmentDriverConfig(input.environment); const parsed = await resolveEnvironmentDriverConfigForRuntime(db, input.companyId, input.environment, { issueId: input.issueId, @@ -1996,6 +2001,9 @@ function createSandboxEnvironmentDriver( }); } } + const resourceBinding = parsed.config.provider === "exe-dev" + ? await readEnvironmentResourceBinding(db, input.environment.id, input.companyId) + : undefined; const acquiredLease = providerLease ?? await pluginWorkerManager.call( pluginProvider.resolved.plugin.id, "environmentAcquireLease", @@ -2005,6 +2013,7 @@ function createSandboxEnvironmentDriver( environmentId: input.environment.id, issueId: input.issueId, config: workerConfig, + ...(resourceBinding ? { resourceBinding } : {}), // Plugin SDK requires a string; ad-hoc test leases use a fresh // UUID so providers that validate or persist the runId still see // a well-formed identifier. @@ -2085,6 +2094,9 @@ function createSandboxEnvironmentDriver( ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}), }; try { + if (parsed.config.provider === "exe-dev") { + await bindEnvironmentResource(db, input.environment.id, input.companyId, acquiredLease.metadata?.environmentResourceBinding); + } return await environmentsSvc.acquireLease({ companyId: input.companyId, environmentId: input.environment.id, @@ -3333,6 +3345,7 @@ function createPluginEnvironmentDriver( driver: "plugin", async acquireRunLease(input) { + await assertExeEnvironmentEnabled(db, input.environment); const parsed = parseEnvironmentDriverConfig(input.environment); if (parsed.driver !== "plugin") { throw new Error(`Expected plugin environment config for driver "${input.environment.driver}".`); diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index e641125b2fc..3687d8a3e11 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1129,9 +1129,14 @@ export function environmentService(db: Db) { values.metadata = withoutManagedEnvironmentArchiveToken(existingMetadata); } + // Client metadata edits cannot remove or overwrite the host's durable VM + // identity. Merge from the current row in SQL to avoid a first-acquire race. + const metadataUpdate = values.metadata !== undefined + ? sql`CASE WHEN ${environments.metadata}->'environmentResourceBinding' IS NOT NULL THEN coalesce(${values.metadata === null ? null : JSON.stringify(values.metadata)}::jsonb, '{}'::jsonb) || jsonb_build_object('environmentResourceBinding', ${environments.metadata}->'environmentResourceBinding') ELSE ${values.metadata === null ? null : JSON.stringify(values.metadata)}::jsonb END` + : undefined; const row = await writeDb .update(environments) - .set(values) + .set({ ...values, ...(metadataUpdate ? { metadata: metadataUpdate } : {}) }) .where(eq(environments.id, id)) .returning() .then((rows) => rows[0] ?? null) diff --git a/server/src/services/exe-environment-gate.ts b/server/src/services/exe-environment-gate.ts new file mode 100644 index 00000000000..ab190f66ad7 --- /dev/null +++ b/server/src/services/exe-environment-gate.ts @@ -0,0 +1,19 @@ +import type { Db } from "@paperclipai/db"; +import { forbidden } from "../errors.js"; +import { instanceSettingsService } from "./instance-settings.js"; + +export function isExeEnvironment(environment: { driver: string; config?: unknown }): boolean { + const config = environment.config as Record | null; + return (environment.driver === "sandbox" && config?.provider === "exe-dev") || + (environment.driver === "plugin" && config?.driverKey === "exe-dev"); +} + +/** Gate new work, never the teardown of an already acquired lease. */ +export async function assertExeEnvironmentEnabled(db: Db, environment: { driver: string; config?: unknown }) { + if (environment.driver === "plugin" && isExeEnvironment(environment)) { + throw forbidden("Configure exe.dev using the sandbox environment driver."); + } + if (isExeEnvironment(environment) && !(await instanceSettingsService(db).getExperimental()).enableExeEnvironments) { + throw forbidden("Enable experimental exe.dev environments in instance settings before using this environment."); + } +} diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index b3d5749d865..9675906ac2b 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -224,6 +224,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta if (parsed.success) { return { enableEnvironments: parsed.data.enableEnvironments ?? false, + enableExeEnvironments: parsed.data.enableExeEnvironments ?? false, enableNativeRunner: parsed.data.enableNativeRunner ?? true, enableManagedSandboxOnly: parsed.data.enableManagedSandboxOnly ?? false, enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false, @@ -267,6 +268,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta } return { enableEnvironments: false, + enableExeEnvironments: false, enableNativeRunner: true, enableManagedSandboxOnly: false, enableIsolatedWorkspaces: false, diff --git a/tests/exe-dev-live/preview.ts b/tests/exe-dev-live/preview.ts new file mode 100644 index 00000000000..62908e3323b --- /dev/null +++ b/tests/exe-dev-live/preview.ts @@ -0,0 +1,50 @@ +/** Interactive private Vite acceptance fixture. Always finish with cleanup. */ +import { randomUUID } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import plugin from "../../packages/plugins/sandbox-providers/exe-dev/src/plugin.js"; +import { ssh, quote } from "../../packages/plugins/sandbox-providers/exe-dev/src/transport.js"; +import type { PluginEnvironmentLease } from "../../packages/plugins/sdk/src/protocol.js"; + +if (process.env.EXE_DEV_LIVE_SMOKE !== "1") throw new Error("EXE_DEV_LIVE_SMOKE=1 required"); +const statePath = process.env.EXE_DEV_PREVIEW_STATE ?? "/tmp/paperclip-exe-preview.json"; +const action = process.argv[2] ?? "create"; +const hooks = plugin.definition; +const sshConfig = { sshPrivateKey: process.env.EXE_DEV_SSH_PRIVATE_KEY, strictHostKeyChecking: "accept-new" as const, timeoutMs: 300000 }; +type State = { companyId: string; environmentId: string; vmName: string; image: string; lease: PluginEnvironmentLease; createdAt: string }; +let state: State; +if (action === "create") { + const image = process.env.PAPERCLIP_E2E_EXE_IMAGE; + if (!image) throw new Error("PAPERCLIP_E2E_EXE_IMAGE required"); + state = { companyId: randomUUID(), environmentId: randomUUID(), vmName: `paperclip-preview-${randomUUID().slice(0, 12)}`, image, lease: { providerLeaseId: null }, createdAt: new Date().toISOString() }; + // Record the owned name before provisioning, including partial failures. + await writeFile(statePath, JSON.stringify(state, null, 2), { mode: 0o600, flag: "wx" }); +} else state = JSON.parse(await readFile(statePath, "utf8")); +if (!/^paperclip-preview-[a-f0-9-]{12}$/.test(state.vmName)) throw new Error("Not an owned preview fixture"); +const config = { ...sshConfig, mode: "create", vmName: state.vmName, image: state.image, reuseLease: true }; +const scope = { companyId: state.companyId, environmentId: state.environmentId, driverKey: "exe-dev", config }; +const host = `${state.vmName}.exe.xyz`; +const run = (script: string) => ssh(sshConfig, host, script); +try { + if (action === "create") { + state.lease = await hooks.onEnvironmentAcquireLease!({ ...scope, agentId: "vite-preview", runId: randomUUID() }); + await writeFile(statePath, JSON.stringify(state, null, 2), { mode: 0o600 }); + const cwd = String(state.lease.metadata?.remoteCwd); + const files = { + "package.json": JSON.stringify({ private: true, type: "module", dependencies: { vite: "8.2.2" } }), + "index.html": 'Paperclip exe.dev preview

Durable Vite preview

', + "main.js": "document.querySelector('#status').textContent='Version 1 — live on exe.dev'; if(import.meta.hot) import.meta.hot.accept();", + "vite.config.js": `export default {server:{host:'0.0.0.0',port:8000,strictPort:true,allowedHosts:[${JSON.stringify(host)}]}};`, + }; + const write = `const fs=require('fs');for(const [name,body] of Object.entries(${JSON.stringify(files)}))fs.writeFileSync(${JSON.stringify(cwd)}+'/'+name,body);`; + await run(`node -e ${quote(write)} && cd ${quote(cwd)} && npm install --ignore-scripts --no-audit --no-fund`); + const unit = `[Unit]\nDescription=Paperclip exe.dev Vite acceptance fixture\nAfter=network.target\n[Service]\nUser=exedev\nWorkingDirectory=${cwd}\nExecStart=/usr/local/bin/node ${cwd}/node_modules/vite/bin/vite.js\nRestart=on-failure\n[Install]\nWantedBy=multi-user.target\n`; + await run(`printf %s ${quote(unit)} | sudo -n tee /etc/systemd/system/paperclip-vite-preview.service >/dev/null && sudo -n systemctl daemon-reload && sudo -n systemctl enable --now paperclip-vite-preview.service`); + } else if (action === "cancel") { + await hooks.onEnvironmentDestroyLease!({ ...scope, providerLeaseId: state.lease.providerLeaseId, leaseMetadata: state.lease.metadata }); + } else if (action === "edit") { + await run(`printf %s ${quote("document.querySelector('#status').textContent='Version 2 — HMR after agent cancellation'; if(import.meta.hot) import.meta.hot.accept();")} > ${quote(String(state.lease.metadata?.remoteCwd) + "/main.js")}`); + } else if (action === "cleanup") { + await ssh(sshConfig, "exe.dev", `rm --json ${quote(state.vmName)}`); + } else throw new Error("Use create, cancel, edit, or cleanup"); + console.log(JSON.stringify({ action, url: `https://${host}`, statePath, createdAt: state.createdAt })); +} finally { await hooks.onShutdown?.(); } diff --git a/tests/exe-dev-live/stress.ts b/tests/exe-dev-live/stress.ts new file mode 100644 index 00000000000..646bcb40f3a --- /dev/null +++ b/tests/exe-dev-live/stress.ts @@ -0,0 +1,172 @@ +/** Explicit paid/provider stress campaign. Never runs under pnpm test. */ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import plugin from "../../packages/plugins/sandbox-providers/exe-dev/src/plugin.js"; +import { openSsh, ssh, quote } from "../../packages/plugins/sandbox-providers/exe-dev/src/transport.js"; +import { prepareAdapterExecutionTargetRuntime, type AdapterSandboxExecutionTarget } from "../../packages/adapter-utils/src/execution-target.js"; +import type { PluginEnvironmentLease } from "../../packages/plugins/sdk/src/protocol.js"; + +if (process.env.EXE_DEV_LIVE_SMOKE !== "1") throw new Error("Set EXE_DEV_LIVE_SMOKE=1 to authorize this real VM campaign"); +const image = process.env.PAPERCLIP_E2E_EXE_IMAGE; +const key = process.env.EXE_DEV_SSH_PRIVATE_KEY; +if (!image || !key) throw new Error("PAPERCLIP_E2E_EXE_IMAGE and EXE_DEV_SSH_PRIVATE_KEY are required"); +const campaign = randomUUID().slice(0, 12); +const vmName = `paperclip-stress-${campaign}`; +const config = { mode: "create", vmName, image, sshPrivateKey: key, reuseLease: true, timeoutMs: 300000 }; +const transport = { ...config, strictHostKeyChecking: "accept-new" as const }; +const scope = { companyId: randomUUID(), environmentId: randomUUID(), driverKey: "exe-dev", config }; +const hooks = plugin.definition; +const leases: PluginEnvironmentLease[] = []; +const checks: { name: string; durationMs: number }[] = []; +const temporary = await mkdtemp(path.join(os.tmpdir(), "paperclip-exe-stress-")); +const outputPath = process.env.EXE_DEV_EVIDENCE_PATH ?? path.join(temporary, "result.json"); +async function check(name: string, run: () => Promise) { + const started = Date.now(); await run(); checks.push({ name, durationMs: Date.now() - started }); console.log(`PASS ${name}`); +} +async function execute(lease: PluginEnvironmentLease, script: string, timeoutMs = 30000) { + return hooks.onEnvironmentExecute!({ ...scope, lease, command: "sh", args: ["-c", script], timeoutMs }); +} +let failure: string | undefined; +try { + await check("compatible image and initial durable VM", async () => { + leases.push(await hooks.onEnvironmentAcquireLease!({ ...scope, runId: randomUUID(), agentId: "agent-0" })); + assert.equal(leases[0].metadata?.vmName, vmName); + }); + const resourceBinding = leases[0].metadata?.environmentResourceBinding as NonNullable>[0]["resourceBinding"]>; + await check("eight independent leases share the bound VM", async () => { + leases.push(...await Promise.all(Array.from({ length: 7 }, (_, i) => hooks.onEnvironmentAcquireLease!({ ...scope, resourceBinding, runId: randomUUID(), agentId: `agent-${i + 1}` })))); + assert.equal(new Set(leases.map((l) => l.providerLeaseId)).size, 8); + assert.equal(new Set(leases.map((l) => l.metadata?.remoteHome)).size, 8); + assert.equal(new Set(leases.map((l) => l.metadata?.bindingId)).size, 1); + }); + await check("64 concurrent commands preserve stdin, output, homes, and exit codes", async () => { + const results = await Promise.all(Array.from({ length: 64 }, async (_, i) => { + const lease = leases[i % leases.length]; + const result = await hooks.onEnvironmentExecute!({ ...scope, lease, command: "node", args: ["-e", `let x='';process.stdin.setEncoding('utf8');process.stdin.on('data',d=>x+=d);process.stdin.on('end',()=>{require('fs').writeFileSync(process.env.HOME+'/result-${i}',x);process.stdout.write(x);process.exit(${i % 3})})`], stdin: `task-${i}-é-🚀\n`.repeat(500), timeoutMs: 60000 }); + assert.equal(result.stdout, `task-${i}-é-🚀\n`.repeat(500)); assert.equal(result.exitCode, i % 3); + return result; + })); + assert.equal(results.length, 64); + }); + await check("large output is completely drained before the exit receipt", async () => { + const result = await execute(leases[0], "node -e \"process.stdout.write('x'.repeat(8*1024*1024))\""); + assert.equal(result.exitCode, 0); + assert.equal(result.stdout.length, 8 * 1024 * 1024); + }); + await check("SSH loss does not kill remote work or imply a termination receipt", async () => { + const root = String(leases[0].metadata?.remoteCwd).replace(/\/workspace$/, ""); + const { child } = await openSsh(transport, `${vmName}.exe.xyz`, `node /opt/paperclip-exe/lease.mjs call ${quote(root)}`); + let ready = false; + child.stdout.on("data", () => { ready = true; }); child.stderr.resume(); + child.stdin.write(JSON.stringify({ type: "start", command: "sh", args: ["-c", "printf started; sleep 2; printf completed > survived-disconnect"], timeoutMs: 10000 }) + "\n"); + const deadline = Date.now() + 15000; + while (!ready && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 100)); + assert.ok(ready, "remote command started"); + child.kill("SIGKILL"); + const result = await execute(leases[0], "sleep 3; cat survived-disconnect"); + assert.equal(result.stdout, "completed"); + }); + await check("workspace seed, remote edits between runs, and copyback", async () => { + const local = path.join(temporary, "workspace"); await mkdir(local); + await writeFile(path.join(local, "initial.txt"), "host seed\n"); + const lease = leases[1]; + const target: AdapterSandboxExecutionTarget = { kind: "remote" as const, transport: "sandbox" as const, providerKey: "exe-dev", remoteCwd: String(lease.metadata?.remoteCwd), runner: { + execute: async (command) => { const result = await hooks.onEnvironmentExecute!({ ...scope, lease, ...command }); return { ...result, signal: result.signal ?? null, pid: null, startedAt: null }; }, + } }; + const first = await prepareAdapterExecutionTargetRuntime({ runId: randomUUID(), target, adapterKey: "stress", workspaceLocalDir: local }); + await first.restoreWorkspace(); + await execute(lease, "printf 'changed remotely\\n' > initial.txt; printf 'durable untracked\\n' > remote-only.txt"); + const next = await prepareAdapterExecutionTargetRuntime({ runId: randomUUID(), target, adapterKey: "stress", workspaceLocalDir: local }); + assert.equal((await execute(lease, "cat initial.txt")).stdout, "changed remotely\n"); + await next.restoreWorkspace(); + assert.equal(await readFile(path.join(local, "initial.txt"), "utf8"), "changed remotely\n"); + assert.equal(await readFile(path.join(local, "remote-only.txt"), "utf8"), "durable untracked\n"); + }); + await check("normal release and worker restart preserve disk", async () => { + const lease = leases[1]; + await hooks.onEnvironmentReleaseLease!({ ...scope, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata }); + await hooks.onShutdown!(); + const resumed = await hooks.onEnvironmentResumeLease!({ ...scope, providerLeaseId: lease.providerLeaseId!, leaseMetadata: lease.metadata }); + assert.equal(resumed.providerLeaseId, lease.providerLeaseId); + assert.equal((await execute(resumed, "cat remote-only.txt")).stdout, "durable untracked\n"); + }); + await check("timeout returns evidence and independent control commands still work", async () => { + const result = await execute(leases[2], "sleep 30", 200); + assert.equal(result.timedOut, true); + assert.equal((await execute(leases[2], "printf healthy")).stdout, "healthy"); + }); + await check("cancel one lease while another continues", async () => { + const pending = execute(leases[3], "sleep 60").catch((error: Error) => error); + const other = execute(leases[4], "sleep 3; printf survived"); + await new Promise((resolve) => setTimeout(resolve, 1000)); + const receipt = await hooks.onEnvironmentReleaseLease!({ ...scope, providerLeaseId: leases[3].providerLeaseId, leaseMetadata: leases[3].metadata, cancelActiveWork: true }); + assert.equal(receipt?.state, "stopped"); + await pending; assert.equal((await other).stdout, "survived"); + }); + await check("lease teardown kills a descendant that escapes its process group", async () => { + const lease = leases[6]; + await execute(lease, "node -e \"const c=require('child_process').spawn('sleep',['120'],{detached:true,stdio:'ignore'});require('fs').writeFileSync('escaped.pid',String(c.pid));c.unref()\""); + const pid = Number((await execute(lease, "cat escaped.pid")).stdout.trim()); + assert.ok(Number.isInteger(pid) && pid > 1); + await hooks.onEnvironmentDestroyLease!({ ...scope, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata }); + assert.equal((await execute(leases[4], `if kill -0 ${pid} 2>/dev/null; then printf alive; else printf stopped; fi`)).stdout, "stopped"); + }); + await check("separately managed HTTP service survives lease teardown", async () => { + const unit = `paperclip-preview-${campaign}`; + const host = `${vmName}.exe.xyz`; + await ssh(transport, host, `sudo -n systemd-run --unit=${quote(unit)} --uid=exedev --collect node -e ${quote("require('http').createServer((q,s)=>s.end('preview-alive')).listen(8000,'0.0.0.0')")}`); + await hooks.onEnvironmentDestroyLease!({ ...scope, providerLeaseId: leases[5].providerLeaseId, leaseMetadata: leases[5].metadata }); + assert.equal((await ssh(transport, host, "curl -fsS http://127.0.0.1:8000")).trim(), "preview-alive"); + const response = await fetch(`https://${host}`, { redirect: "manual" }); + assert.ok([302, 303, 307, 308, 401, 403].includes(response.status), `Preview must remain private; got ${response.status}`); + if (response.status < 400) { + const login = new URL(response.headers.get("location")!, `https://${host}`); + assert.equal(login.origin, `https://${host}`); + assert.equal(login.pathname, "/__exe.dev/login"); + } + await ssh(transport, host, `sudo -n systemctl stop ${quote(unit)}`); + }); + const soakMs = Number(process.env.EXE_DEV_SOAK_MS ?? 0); + if (soakMs > 0) await check(`idle persistence after ${soakMs}ms`, async () => { + await new Promise((resolve) => setTimeout(resolve, soakMs)); + assert.equal((await execute(leases[1], "cat remote-only.txt")).stdout, "durable untracked\n"); + }); + await check("VM reboot preserves disk and resumed lease identity", async () => { + const host = `${vmName}.exe.xyz`; + const previousBoot = await ssh(transport, host, "cat /proc/sys/kernel/random/boot_id"); + await ssh(transport, host, "sudo -n systemctl reboot").catch(() => {}); + const deadline = Date.now() + 90000; + let rebooted = false; + while (Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 2000)); + try { if (await ssh({ ...transport, timeoutMs: 10000 }, host, "cat /proc/sys/kernel/random/boot_id") !== previousBoot) { rebooted = true; break; } } catch {} + } + assert.ok(rebooted, "VM completed reboot"); + const lease = leases[1]; + const resumed = await hooks.onEnvironmentResumeLease!({ ...scope, providerLeaseId: lease.providerLeaseId!, leaseMetadata: lease.metadata }); + assert.equal(resumed.metadata?.bindingId, lease.metadata?.bindingId); + assert.equal((await execute(resumed, "cat remote-only.txt")).stdout, "durable untracked\n"); + }); +} catch (error) { + failure = error instanceof Error ? error.stack : String(error); +} finally { + for (const lease of leases) { + try { await hooks.onEnvironmentDestroyLease!({ ...scope, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata }); } + catch (error) { failure ??= `Lease cleanup failed: ${String(error)}`; } + } + // Only this explicitly generated campaign VM is removable. Durable production + // lifecycle hooks never call rm. Missing VMs are already cleaned up. + try { + const list = JSON.parse(await ssh(transport, "exe.dev", "ls --json")); + if (list.vms.some((vm: Record) => vm.vm_name === vmName)) await ssh(transport, "exe.dev", `rm --json ${quote(vmName)}`); + const after = JSON.parse(await ssh(transport, "exe.dev", "ls --json")); + assert.ok(!after.vms.some((vm: Record) => vm.vm_name === vmName)); + } catch (error) { failure ??= `VM cleanup failed: ${String(error)}`; } + await hooks.onShutdown?.(); + await writeFile(outputPath, JSON.stringify({ campaign, image, vmName, checks, failure, finishedAt: new Date().toISOString() }, null, 2), { mode: 0o600 }); + if (failure) { console.error(failure); process.exitCode = 1; } + console.log(`Evidence: ${outputPath}`); +} diff --git a/tests/exe-dev-live/tsconfig.json b/tests/exe-dev-live/tsconfig.json new file mode 100644 index 00000000000..0b710b4cfde --- /dev/null +++ b/tests/exe-dev-live/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "noEmit": true, "rootDir": "../..", "types": ["node"], "typeRoots": ["../../cli/node_modules/@types"] }, + "include": ["./**/*.ts"] +} diff --git a/tests/runner-e2e/billing.test.ts b/tests/runner-e2e/billing.test.ts index 7a5e54d9049..ffc8dbc6da0 100644 --- a/tests/runner-e2e/billing.test.ts +++ b/tests/runner-e2e/billing.test.ts @@ -28,6 +28,11 @@ function result(overrides: Partial = {}): RunnerE2EResult { } describe("runner E2E billing summaries", () => { + it("does not price shared exe.dev leases as Daytona resources", () => { + const usage = buildRuntimeUsage({ environmentId: "exe-dev", runs: [], leases: [] }); + expect(usage).toMatchObject({ provider: "exe-dev", costStatus: "unavailable", costSource: "provider_cost_unavailable" }); + expect(usage.estimatedListCostUsd).toBeUndefined(); + }); it("summarizes provider-reported token usage and cost", () => { const billing = summarizeExecutionBilling( result({ diff --git a/tests/runner-e2e/billing.ts b/tests/runner-e2e/billing.ts index 71729e77bb5..0cb7d574c06 100644 --- a/tests/runner-e2e/billing.ts +++ b/tests/runner-e2e/billing.ts @@ -169,6 +169,15 @@ export function buildRuntimeUsage(input: { } const leases = input.leases ?? []; + if (input.environmentId === "exe-dev") { + // exe.dev bills pooled subscription resources, not these individual leases. + // Never apply Daytona's per-second resource prices to a shared durable VM. + return { + provider: "exe-dev", agentRunDurationMs, leaseCount: leases.length, + leaseDurationMs: leases.length ? leases.reduce((sum, lease) => sum + durationBetween(lease.acquiredAt, lease.releasedAt ?? input.fallbackFinishedAt ?? lease.updatedAt ?? null), 0) : null, + costStatus: "unavailable", costSource: "provider_cost_unavailable", + }; + } let leaseDurationMs = 0; let estimatedListCostUsd = 0; let resourcesComplete = leases.length > 0; diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 45944727d14..cd2ad82f1fe 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -41,10 +41,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 22, 35, 52, 24, 42, 14, 10, 2, + 22, 35, 52, 24, 21, 4, 42, 14, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(201); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(201); + expect(validateRunnerCatalog()).toHaveLength(226); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(226); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); @@ -72,7 +72,7 @@ describe("runner E2E catalog", () => { expect( runnerTasks.find((task) => task.id === "plan-revise-accept") ?.attemptTimeoutMs, - ).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000 }); + ).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000, "exe-dev": 12 * 60_000 }); }); it("defines the warm Daytona continuity fixture as exactly two Codex cells", () => { diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 7e6cc46996e..026d4193860 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -26,12 +26,13 @@ import type { SecretReference, } from "./types.js"; -const ENVIRONMENT_IDS = ["local", "daytona"] as const; +const ENVIRONMENT_IDS = ["local", "daytona", "exe-dev"] as const; const SELECTABLE_GROUPS = [ "legacy", "native", "local", "daytona", + "exe-dev", "warm", "core", "breadth", @@ -389,6 +390,26 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ }, ] as const; +export const exeEnvironment: EnvironmentFixture = { + id: "exe-dev", label: "Durable exe.dev VM", groups: ["exe-dev"], + driver: "sandbox", provider: "exe-dev", credential: "EXE_DEV_SSH_PRIVATE_KEY", + lifecycle: { setup: "create_via_api", probe: "run_context_via_api", cleanup: "delete_via_api_and_destroy_leases" }, + expectedExecutionTarget: { kind: "remote", transport: "sandbox" }, + buildEnvironment(input) { + if (!isImmutableDaytonaImage(input.exeImage)) throw new Error("PAPERCLIP_E2E_EXE_IMAGE must be an immutable image digest"); + const key = input.secretRefs.EXE_DEV_SSH_PRIVATE_KEY; + if (!key) throw new Error("EXE_DEV_SSH_PRIVATE_KEY secret reference required"); + return { + name: `Runner E2E exe.dev ${input.executionId}`, driver: "sandbox", + config: { provider: "exe-dev", mode: "create", image: input.exeImage, + vmName: `paperclip-e2e-${createHash("sha256").update(input.executionId).digest("hex").slice(0, 20)}`, + sshPrivateKey: key.secretId, ...(input.secretRefs.EXE_DEV_REGISTRY_AUTH ? { registryAuth: input.secretRefs.EXE_DEV_REGISTRY_AUTH.secretId } : {}), reuseLease: true, runnerLifecycleMode: "per_turn", + cpu: 2, memory: "4GB", disk: "20GB", timeoutMs: 300000 }, + envVars: {}, + }; + }, +}; + export const daytonaWarmEnvironment: EnvironmentFixture = { id: "daytona", configurationKey: "warm-reuse-v1", @@ -443,7 +464,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ expectedRunCount: 1, attemptTimeoutMs: { local: 8 * 60_000, - daytona: 15 * 60_000, + daytona: 15 * 60_000, "exe-dev": 15 * 60_000, }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `Runner E2E PAPERCLIP_E2E_OK_${nonce}`, @@ -492,7 +513,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ expectedRunCount: 3, attemptTimeoutMs: { local: 8 * 60_000, - daytona: 12 * 60_000, + daytona: 12 * 60_000, "exe-dev": 12 * 60_000, }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `Runner E2E plan lifecycle ${nonce}`, @@ -560,7 +581,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ expectedRunCount: 1, attemptTimeoutMs: { local: 8 * 60_000, - daytona: 15 * 60_000, + daytona: 15 * 60_000, "exe-dev": 15 * 60_000, }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `Runner E2E ask mode ${nonce}`, @@ -633,7 +654,7 @@ const structuredQuestionResumeTask = { workMode: "standard", flow: "question_resume_completion", expectedRunCount: 2, - attemptTimeoutMs: { local: 12 * 60_000, daytona: 12 * 60_000 }, + attemptTimeoutMs: { local: 12 * 60_000, daytona: 12 * 60_000, "exe-dev": 12 * 60_000 }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `Runner E2E structured question ${nonce}`, buildVisibleMarker: (nonce) => `PAPERCLIP_E2E_QUESTION_DONE_${nonce}`, @@ -703,7 +724,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ workMode: "standard", flow: "single_turn", expectedRunCount: 1, - attemptTimeoutMs: { local: 8 * 60_000, daytona: 8 * 60_000 }, + attemptTimeoutMs: { local: 8 * 60_000, daytona: 8 * 60_000, "exe-dev": 8 * 60_000 }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `OpenRouter breadth hello ${nonce}`, buildVisibleMarker: (nonce) => breadthMarker("H", nonce), @@ -724,7 +745,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ workMode: "standard", flow: "question_resume_completion", expectedRunCount: 2, - attemptTimeoutMs: { local: 12 * 60_000, daytona: 12 * 60_000 }, + attemptTimeoutMs: { local: 12 * 60_000, daytona: 12 * 60_000, "exe-dev": 12 * 60_000 }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `OpenRouter breadth question ${nonce}`, buildVisibleMarker: (nonce) => breadthMarker("Q_C", nonce), @@ -751,7 +772,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ workMode: "planning", flow: "plan_approval_completion", expectedRunCount: 2, - attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000 }, + attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000, "exe-dev": 15 * 60_000 }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `OpenRouter breadth plan ${nonce}`, buildVisibleMarker: (nonce) => breadthMarker("P_OK", nonce), @@ -820,7 +841,7 @@ export const daytonaWarmContinuityTask: RunnerTaskFixture = { workMode: "standard", flow: "warm_three_turn", expectedRunCount: 3, - attemptTimeoutMs: { local: 30 * 60_000, daytona: 30 * 60_000 }, + attemptTimeoutMs: { local: 30 * 60_000, daytona: 30 * 60_000, "exe-dev": 30 * 60_000 }, turnTimeoutMs: 10 * 60_000, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `Runner E2E warm Daytona continuity ${nonce}`, @@ -871,7 +892,7 @@ export const connectionReviewSuite: RunnerSuiteFixture = { tasks: (["approve", "decline", "always", "restart"] as const).map(decision => ({ id: `tool-review-${decision}`, label: `Connection review: ${decision}`, groups: [], workMode: "standard", flow: "governed_tool_review", toolReviewDecision: decision, - expectedRunCount: 2, attemptTimeoutMs: { local: 12 * 60_000, daytona: 12 * 60_000 }, + expectedRunCount: 2, attemptTimeoutMs: { local: 12 * 60_000, daytona: 12 * 60_000, "exe-dev": 12 * 60_000 }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: nonce => `Runner E2E connection review ${decision} ${nonce}`, buildVisibleMarker: nonce => `PAPERCLIP_E2E_REVIEW_DONE_${nonce}`, @@ -929,6 +950,20 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ definitionMetadata: { version: 1, resetRunsCountedSeparately: true }, }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), + { + id: "exe-compatibility", label: "Experimental exe.dev Compatibility", + description: "Seven legacy/native profiles across three workflows on prebuilt durable VMs.", + groups: ["exe-dev", "core"], manualOnly: true, + profiles: runnerProfiles.map((profile) => ({ ...profile, supportedEnvironments: [...profile.supportedEnvironments, "exe-dev" as const] })), + environments: [exeEnvironment], tasks: runnerTasks, expectedMatrixSize: 21, + }, + { + id: "exe-recovery", label: "Experimental exe.dev Recovery", + description: "Question/resume and controller restart on durable legacy and native Codex workspaces.", + groups: ["exe-dev"], manualOnly: true, + profiles: runnerProfiles.filter((profile) => ["legacy-codex", "runner-codex"].includes(profile.id)).map((profile) => ({ ...profile, supportedEnvironments: [...profile.supportedEnvironments, "exe-dev" as const] })), + environments: [exeEnvironment], tasks: localIntegrityTasks, expectedMatrixSize: 4, + }, { id: "core-compatibility", label: "Core Runner Compatibility", diff --git a/tests/runner-e2e/chat-cases.ts b/tests/runner-e2e/chat-cases.ts index a3fcf4cbbd6..e02aa44aa79 100644 --- a/tests/runner-e2e/chat-cases.ts +++ b/tests/runner-e2e/chat-cases.ts @@ -26,7 +26,7 @@ export const chatTasks: readonly RunnerTaskFixture[] = CHAT_CASES.map( flow: "agent_chat", workMode: "standard", expectedRunCount, // Provider turns, including cancelled turns and handed-off work; reset runs are separate. - attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000 }, + attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000, "exe-dev": 15 * 60_000 }, expectedTerminalState: { issue: "in_review", run: "succeeded" }, buildTitle: (nonce) => `Chat acceptance ${id} ${nonce}`, buildPrompt: (nonce) => `Let's discuss ${nonce}.`, diff --git a/tests/runner-e2e/everyday-cases.ts b/tests/runner-e2e/everyday-cases.ts index cb562cab4ae..edcf0430865 100644 --- a/tests/runner-e2e/everyday-cases.ts +++ b/tests/runner-e2e/everyday-cases.ts @@ -96,7 +96,7 @@ export const everydayTasks: readonly RunnerTaskFixture[] = definitions.map( workMode: "standard", flow: "everyday_workflow", expectedRunCount, - attemptTimeoutMs: { local: 12 * 60_000, daytona: 30 * 60_000 }, + attemptTimeoutMs: { local: 12 * 60_000, daytona: 30 * 60_000, "exe-dev": 30 * 60_000 }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: (nonce) => `${label} ${nonce}`, buildPrompt: () => prompt, diff --git a/tests/runner-e2e/exe-shared-agents.ts b/tests/runner-e2e/exe-shared-agents.ts new file mode 100644 index 00000000000..faf4aea0889 --- /dev/null +++ b/tests/runner-e2e/exe-shared-agents.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { expect, type Page } from "@playwright/test"; +import { RunnerApi, pollUntil } from "./api.js"; +import { runnerProfiles, runnerTasks } from "./catalog.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; + +/** Real native/native and native/legacy overlap on the same durable VM. */ +export async function verifyExeSharedAgents(input: { api: RunnerApi; page: Page; fixtures: LiveFixtureValues; nonce: string; workspacePath: string }) { + const { api, fixtures } = input; + const task = runnerTasks.find((entry) => entry.id === "message-marker")!; + const agents = await Promise.all(["runner-codex", "runner-codex", "legacy-codex"].map(async (id, i) => { + const profile = runnerProfiles.find((entry) => entry.id === id)!; + const nonce = `${input.nonce}-shared-${i}`; + const agent = await api.post<{ id: string }>(`/api/companies/${fixtures.company.id}/agents`, profile.buildAgent({ + environmentId: fixtures.environment.id, environmentFixtureId: "exe-dev", workspacePath: input.workspacePath, + secretRefs: fixtures.secretRefs, executionId: nonce, + })); + return { id: agent.id, profile: id, nonce }; + })); + type Run = { id: string; status: string; startedAt: string; finishedAt: string; error?: string; contextSnapshot?: { issueId?: string; taskId?: string } }; + const issueRuns = async (issueId: string, agentId: string) => + (await api.get(`/api/companies/${fixtures.company.id}/heartbeat-runs?agentId=${agentId}&limit=20`)) + .filter((run) => run.contextSnapshot?.issueId === issueId || run.contextSnapshot?.taskId === issueId); + const issues = await Promise.all(agents.map(async (agent) => ({ agent, issue: await api.post<{ id: string; identifier: string }>(`/api/companies/${fixtures.company.id}/issues`, { + title: task.buildTitle(agent.nonce), description: task.buildPrompt(agent.nonce), status: "todo", assigneeAgentId: agent.id, + }) }))); + try { + const results = await Promise.all(issues.map(async ({ agent, issue }) => { + const result = await pollUntil({ label: `shared ${agent.profile} task`, deadlineAt: Date.now() + 10 * 60_000, intervalMs: 1000, + load: async () => ({ issue: await api.get<{ status: string }>(`/api/issues/${issue.id}`), runs: await issueRuns(issue.id, agent.id) }), + accept: (value) => value.issue.status === "done" && value.runs.some((run) => run.status === "succeeded") && value.runs.every((run) => ["succeeded", "failed", "cancelled", "timed_out"].includes(run.status)), + reject: (value) => value.runs.some((run) => ["failed", "timed_out"].includes(run.status)) ? `Shared agent failed: ${value.runs.map((run) => run.error ?? run.status).join("; ")}` : undefined, + }); + return { agentId: agent.id, profile: agent.profile, issueId: issue.id, runs: result.runs }; + })); + const firstRuns = results.map((result) => result.runs.find((run) => run.status === "succeeded")!); + assert.ok(Math.max(...firstRuns.map((run) => Date.parse(run.startedAt))) < Math.min(...firstRuns.map((run) => Date.parse(run.finishedAt))), "All three real agent executions overlapped"); + const leases = await api.get<{ issueId: string; metadata: Record }[]>(`/api/environments/${fixtures.environment.id}/leases`); + const shared = leases.filter((lease) => issues.some(({ issue }) => issue.id === lease.issueId)); + assert.equal(new Set(shared.map((lease) => lease.issueId)).size, 3); + assert.equal(new Set(shared.map((lease) => lease.metadata.vmName)).size, 1); + assert.equal(new Set(shared.map((lease) => lease.metadata.remoteHome)).size, 3); + assert.equal(new Set(shared.map((lease) => lease.metadata.remoteCwd)).size, 3); + for (const { agent, issue } of issues) { + await input.page.goto(`${api.baseURL}/${fixtures.company.issuePrefix}/issues/${issue.identifier}`); + await expect(input.page.getByText(task.buildVisibleMarker(agent.nonce), { exact: true }).first()).toBeVisible({ timeout: 30000 }); + } + return { results, vmName: shared[0].metadata.vmName, independentHomes: 3, independentWorkspaces: 3 }; + } finally { + for (const { issue, agent } of issues) { + const runs = await issueRuns(issue.id, agent.id); + for (const run of runs) if (["queued", "running"].includes(run.status)) await api.post(`/api/heartbeat-runs/${run.id}/cancel`); + } + } +} diff --git a/tests/runner-e2e/first-task-cases.ts b/tests/runner-e2e/first-task-cases.ts index 8244f0ead12..e63e294c47c 100644 --- a/tests/runner-e2e/first-task-cases.ts +++ b/tests/runner-e2e/first-task-cases.ts @@ -76,7 +76,7 @@ export const firstTaskTasks: readonly RunnerTaskFixture[] = flow: "first_task", workMode: "standard", expectedRunCount, - attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000 }, + attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000, "exe-dev": 15 * 60_000 }, expectedTerminalState: { issue: "in_review", run: "succeeded" }, buildTitle: (nonce) => `First task ${id} ${nonce}`, buildPrompt: (nonce) => firstTaskScenario(id, nonce).prompt, diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index d1e9cede374..2852aa61206 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -24,7 +24,7 @@ const AMBIENT_EXTERNAL_STATE_KEYS = [ "PAPERCLIP_STORAGE_S3_PREFIX", "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", ] as const; -const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)(?:_|$)/; +const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|EXE_DEV)(?:_|$)/; export function runnerE2EServerControlPaths(temporaryRoot: string) { const controlDirectory = path.join(temporaryRoot, "control"); diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index c159246c885..5fb923aac9d 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1113,6 +1113,9 @@ async function main() { ); } + if (executions.some((execution) => execution.environment.id === "exe-dev") && !isImmutableDaytonaImage(process.env.PAPERCLIP_E2E_EXE_IMAGE)) { + throw new Error("PAPERCLIP_E2E_EXE_IMAGE must be an immutable image digest"); + } const campaignId = cleanId( process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? `local-${new Date().toISOString().replace(/[:.]/g, "-")}`, diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index c7532a54a3c..dfa9fe3e77c 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; +import { ssh, quote } from "../../packages/plugins/sandbox-providers/exe-dev/src/transport.js"; import path from "node:path"; import { FixtureRegistry } from "./fixture-registry.js"; import type { RunnerApi } from "./api.js"; @@ -102,18 +104,19 @@ export async function setupLiveFixtures(input: { workspacePath: string; credentials: Partial>; daytonaImage?: string; + exeImage?: string; }): Promise { const { api, execution } = input; const registry = new FixtureRegistry(); - if (execution.environment.id === "daytona") { + if (execution.environment.driver === "sandbox") { registry.register({ id: "sandbox-provider", async setup() { return api.post("/api/plugins/install", { packageName: path.resolve( import.meta.dirname, - "../../packages/plugins/sandbox-providers/daytona", + `../../packages/plugins/sandbox-providers/${execution.environment.provider}`, ), isLocalPath: true, }); @@ -148,7 +151,9 @@ export async function setupLiveFixtures(input: { async setup(resolved) { const company = value(resolved, "company"); const refs: SecretReferenceMap = {}; - for (const credentialName of execution.requiredCredentials) { + const credentialNames = [...execution.requiredCredentials]; + if (execution.environment.id === "exe-dev" && input.credentials.EXE_DEV_REGISTRY_AUTH) credentialNames.push("EXE_DEV_REGISTRY_AUTH"); + for (const credentialName of credentialNames) { const rawValue = input.credentials[credentialName]; if (!rawValue) throw new Error(`Missing credential ${credentialName}`); const secret = await api.postSensitive( @@ -175,7 +180,7 @@ export async function setupLiveFixtures(input: { dependencies: [ "company", "secrets", - ...(execution.environment.id === "daytona" ? ["sandbox-provider"] : []), + ...(execution.environment.driver === "sandbox" ? ["sandbox-provider"] : []), ], async setup(resolved) { const company = value(resolved, "company"); @@ -201,13 +206,29 @@ export async function setupLiveFixtures(input: { execution.environment.buildEnvironment({ secretRefs, daytonaImage: input.daytonaImage, + exeImage: input.exeImage, executionId: input.executionNonce, }), ); }, - async teardown(environment) { - if (execution.environment.id === "daytona") { + async teardown(environment, resolved) { + if (execution.environment.driver === "sandbox") { await deleteDaytonaEnvironment(api, environment.id); + if (execution.environment.id === "exe-dev") { + const vmName = environment.config?.vmName; + if (typeof vmName !== "string" || !/^paperclip-e2e-[a-f0-9]{20}$/.test(vmName)) throw new Error("Refusing cleanup of a non-campaign VM"); + const sshConfig = { sshPrivateKey: input.credentials.EXE_DEV_SSH_PRIVATE_KEY, strictHostKeyChecking: "accept-new" as const, timeoutMs: 120000 }; + const listing = JSON.parse(await ssh(sshConfig, "exe.dev", "ls -l --json")); + const vm = listing.vms.find((entry: Record) => entry.vm_name === vmName); + if (vm) { + const company = value(resolved, "company"); + const tag = "paperclip-" + createHash("sha256").update(`${company.id}\0${environment.id}`).digest("hex").slice(0, 32); + if (!vm.tags?.includes(tag)) throw new Error("Refusing deletion: VM ownership tag does not match this fixture"); + await ssh(sshConfig, "exe.dev", `rm --json ${quote(vmName)}`); + const remaining = JSON.parse(await ssh(sshConfig, "exe.dev", "ls --json")); + if (remaining.vms.some((entry: Record) => entry.vm_name === vmName)) throw new Error("exe.dev VM cleanup could not be confirmed"); + } + } } }, }); diff --git a/tests/runner-e2e/report-catalog.ts b/tests/runner-e2e/report-catalog.ts index 8d6fcb821be..49711374483 100644 --- a/tests/runner-e2e/report-catalog.ts +++ b/tests/runner-e2e/report-catalog.ts @@ -27,7 +27,7 @@ export function parseReportExecutionId(id: string) { if (typeof id !== "string") throw new Error("Report execution identity must be a string"); const match = - /^([a-z0-9][a-z0-9_-]*)\.([a-z0-9][a-z0-9_-]*)\.(local|daytona)\.([a-z0-9][a-z0-9_-]*)$/.exec( + /^([a-z0-9][a-z0-9_-]*)\.([a-z0-9][a-z0-9_-]*)\.(local|daytona|exe-dev)\.([a-z0-9][a-z0-9_-]*)$/.exec( id, ); if (!match || id.length > 512) @@ -37,7 +37,7 @@ export function parseReportExecutionId(id: string) { return { suiteId: match[1]!, profileId: match[2]!, - environmentId: match[3]! as "local" | "daytona", + environmentId: match[3]! as "local" | "daytona" | "exe-dev", caseId: match[4]!, }; } diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index be7a0368aa8..909776e2ab9 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -81,7 +81,7 @@ const relativeFile: Rule = (value, at) => { invalid(at); }; const runtime = shape({ - provider: oneOf("local", "daytona"), + provider: oneOf("local", "daytona", "exe-dev"), agentRunDurationMs: number, leaseDurationMs: nullable(number), leaseCount: integer, @@ -181,7 +181,7 @@ const fields = { ), error: optional(string), profileId: string, - environmentId: oneOf("local", "daytona"), + environmentId: oneOf("local", "daytona", "exe-dev"), caseId: string, provider: string, model: string, diff --git a/tests/runner-e2e/runner-wss-tunnel.test.ts b/tests/runner-e2e/runner-wss-tunnel.test.ts new file mode 100644 index 00000000000..856ad8a4127 --- /dev/null +++ b/tests/runner-e2e/runner-wss-tunnel.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; +import { allowsRunnerUpgrade } from "./runner-wss-tunnel.js"; +describe("runner-only test tunnel", () => { + it("allows only the exact authenticated runner WebSocket route", () => { + expect(allowsRunnerUpgrade("GET", "/api/runner/v1/connect/1234-abcd")).toBe(true); + for (const url of ["/", "/api/companies", "/api/instance/settings", "/api/runner/v1/connect/../companies", "/api/runner/v1/connect/a?token=x", "/api/runner/v1/connect/a/", "/api/runner/v1/connect/%2fapi"]) { + expect(allowsRunnerUpgrade("GET", url)).toBe(false); + } + expect(allowsRunnerUpgrade("POST", "/api/runner/v1/connect/a")).toBe(false); + }); +}); diff --git a/tests/runner-e2e/runner-wss-tunnel.ts b/tests/runner-e2e/runner-wss-tunnel.ts new file mode 100644 index 00000000000..01c42179d38 --- /dev/null +++ b/tests/runner-e2e/runner-wss-tunnel.ts @@ -0,0 +1,52 @@ +import http from "node:http"; +import net from "node:net"; +import { spawn } from "node:child_process"; + +/** The public tunnel can reach only authenticated PRP upgrades, never board APIs. */ +export function allowsRunnerUpgrade(method: string | undefined, url: string | undefined): boolean { + return method === "GET" && /^\/api\/runner\/v1\/connect\/[A-Za-z0-9_-]+$/.test(url ?? ""); +} + +export async function startRunnerWssTunnel(binary: string, targetPort: number) { + const sockets = new Set(); + const proxy = http.createServer((_req, res) => { res.writeHead(404); res.end(); }); + proxy.on("upgrade", (request, socket, head) => { + if (!allowsRunnerUpgrade(request.method, request.url)) { + socket.end("HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"); return; + } + const upstream = net.connect(targetPort, "127.0.0.1"); + sockets.add(upstream); + upstream.on("close", () => { sockets.delete(upstream); socket.destroy(); }); + upstream.on("error", () => socket.destroy()); + socket.on("error", () => upstream.destroy()); + socket.on("close", () => upstream.destroy()); + upstream.on("connect", () => { + const headers = Object.entries(request.headers).filter(([name]) => name !== "host") + .flatMap(([name, value]) => (Array.isArray(value) ? value : [value]).filter((v) => v !== undefined).map((v) => `${name}: ${v}`)); + upstream.write(`GET ${request.url} HTTP/1.1\r\nHost: 127.0.0.1:${targetPort}\r\n${headers.join("\r\n")}\r\n\r\n`); + if (head.length) upstream.write(head); + socket.pipe(upstream); upstream.pipe(socket); + }); + }); + await new Promise((resolve) => proxy.listen(0, "127.0.0.1", resolve)); + const address = proxy.address() as net.AddressInfo; + const child = spawn(binary, ["tunnel", "--no-autoupdate", "--protocol", "http2", "--url", `http://127.0.0.1:${address.port}`], { stdio: ["ignore", "pipe", "pipe"] }); + const close = async () => { + child.kill("SIGTERM"); + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => proxy.close(() => resolve())); + }; + try { + const publicUrl = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("Runner-only tunnel did not start in 60 seconds")), 60000); + const read = (data: Buffer) => { + const match = data.toString().match(/https:\/\/([a-z0-9-]+\.trycloudflare\.com)/); + if (match) { clearTimeout(timeout); resolve(`wss://${match[1]}`); } + }; + child.stdout.on("data", read); child.stderr.on("data", read); + child.once("error", (error) => { clearTimeout(timeout); reject(error); }); + child.once("exit", () => { clearTimeout(timeout); reject(new Error("Runner-only tunnel exited during startup")); }); + }); + return { publicUrl, close }; + } catch (error) { await close(); throw error; } +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index f9700a8e312..9d493a95957 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,5 +1,6 @@ import { runContinuationFlow } from "./continuation-flow.js"; import { runEverydayFlow } from "./everyday-flow.js"; +import { verifyExeSharedAgents } from "./exe-shared-agents.js"; import { createTaskThroughUi, submitTaskReply } from "./user-actions.js"; import { runFirstTaskFlow, setupFirstTaskFixtures } from "./first-task-flow.js"; @@ -393,11 +394,11 @@ function nativeRunEventIntegrityFailures( const envelope = record(event.payload?.prpEvent); if (Object.keys(envelope).length === 0) continue; if ( - envelope.schema !== "paperclip.prp.event.v1" || - envelope.schemaVersion !== 1 || - event.protocolSchemaVersion !== 1 + ![1, 2].includes(Number(envelope.schemaVersion)) || + envelope.schema !== `paperclip.prp.event.v${envelope.schemaVersion}` || + event.protocolSchemaVersion !== envelope.schemaVersion ) { - failures.push(`run ${run.id} exposed a malformed PRP v1 envelope`); + failures.push(`run ${run.id} exposed a malformed PRP envelope`); } if (envelope.runId !== run.id) { failures.push( @@ -585,14 +586,14 @@ for (const execution of executions) { }; const captureRuntimeLeases = async () => { - if (!fixtures || execution.environment.id !== "daytona") return; + if (!fixtures || execution.environment.driver !== "sandbox") return; const listed = await api.get( `/api/environments/${fixtures.environment.id}/leases`, ); const selectedRunIds = new Set(selectedRuns.map((run) => run.id)); const relevant = listed.filter( (lease) => - lease.provider === "daytona" && + lease.provider === execution.environment.provider && (selectedRunIds.size === 0 || (lease.heartbeatRunId && selectedRunIds.has(lease.heartbeatRunId)) || @@ -745,6 +746,7 @@ for (const execution of executions) { enableNativeRunner: boolean; }>("/api/instance/settings/experimental", { enableNativeRunner: true, + ...(execution.environment.id === "exe-dev" ? { enableExeEnvironments: true, enableEnvironments: true } : {}), ...(["warm_three_turn", "everyday_workflow"].includes(execution.task.flow) ? { enableIsolatedWorkspaces: true } : {}), @@ -764,6 +766,7 @@ for (const execution of executions) { workspacePath, credentials, daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, + exeImage: process.env.PAPERCLIP_E2E_EXE_IMAGE, }); await writeSanitizedJson( @@ -1281,9 +1284,9 @@ for (const execution of executions) { }) .last() .check(); - // Required single-select questions submit as soon as the radio is - // checked; waiting for the multi-answer submit control would race the - // successful continuation and misreport it as a UI failure. + // Selecting an answer only edits the form. Submit it through the same + // explicit control a user must use before the agent can continue. + await page.getByRole("button", { name: "Submit answers", exact: true }).last().click(); questionLifecycleEvidence = { interaction: questionInteraction, answer: expectedAnswer.optionLabel, @@ -1824,7 +1827,7 @@ for (const execution of executions) { const environmentDriver = environmentContext.driver ?? persistedEnvironment.driver; const observedEnvironment = - environmentDriver === "sandbox" ? "daytona" : environmentDriver; + environmentDriver === "sandbox" ? execution.environment.provider : environmentDriver; const observedRuntimeMode = run.runtimeMode ?? (persistedAgent.adapterType === "paperclip_runner" @@ -2239,7 +2242,7 @@ for (const execution of executions) { const expectedTransport = execution.environment.id === "daytona" ? "provider_ingress" - : "local_loopback"; + : execution.environment.id === "exe-dev" ? "direct_outbound" : "local_loopback"; if (selectedTransport?.mode !== expectedTransport) { invariantFailures.push( `Expected native runner transport ${expectedTransport}; observed ${String(selectedTransport?.mode)}`, @@ -2375,6 +2378,10 @@ for (const execution of executions) { `Runtime invariant failure: ${invariantFailures.join("; ")}`, ); } + if (execution.id === "exe-compatibility.runner-codex.exe-dev.message-marker") { + const sharedAgents = await verifyExeSharedAgents({ api, page, fixtures, nonce, workspacePath }); + await writeSanitizedJson(snapshotsDir, "exe-shared-agents.json", sharedAgents, secrets); + } } } catch (error) { primaryError = error; diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index 7412448d555..ee35f491a21 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -1,3 +1,4 @@ +import { startRunnerWssTunnel } from "./runner-wss-tunnel.js"; import { spawn, type ChildProcess } from "node:child_process"; import { createWriteStream } from "node:fs"; import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; @@ -347,7 +348,13 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) { }); } +let runnerTunnel: Awaited> | undefined; async function supervise() { + const tunnelBinary = process.env.PAPERCLIP_E2E_RUNNER_TUNNEL_BIN; + if (tunnelBinary && process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS?.includes(".exe-dev.")) { + runnerTunnel = await startRunnerWssTunnel(tunnelBinary, Number(port)); + definedServerEnvironment.PAPERCLIP_RUNNER_PUBLIC_URL = runnerTunnel.publicUrl; + } const databaseReservation = await prepareRunnerE2EServerConfig({ temporaryRoot, configPath, @@ -401,3 +408,5 @@ try { await new Promise((resolve) => log.end(resolve)); process.exitCode = exitCode; + +await runnerTunnel?.close(); diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 795260ce2d9..c64780de016 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -3,11 +3,13 @@ export const CREDENTIAL_NAMES = [ "ANTHROPIC_API_KEY", "OPENROUTER_API_KEY", "DAYTONA_API_KEY", + "EXE_DEV_SSH_PRIVATE_KEY", + "EXE_DEV_REGISTRY_AUTH", ] as const; export type CredentialName = (typeof CREDENTIAL_NAMES)[number]; export type RunnerGeneration = "legacy" | "native"; -export type RunnerEnvironmentId = "local" | "daytona"; +export type RunnerEnvironmentId = "local" | "daytona" | "exe-dev"; export type RunnerTaskWorkMode = "standard" | "planning" | "ask"; export type RunnerTaskFlow = | "everyday_workflow" @@ -43,6 +45,7 @@ export interface AgentFixtureBuildInput { export interface EnvironmentFixtureBuildInput { secretRefs: SecretReferenceMap; daytonaImage?: string; + exeImage?: string; executionId: string; } @@ -68,7 +71,7 @@ export interface RunnerProfileFixture { capturedAt: string; sourceUrl: string; }; - credential: Exclude; + credential: Exclude; supportedEnvironments: readonly RunnerEnvironmentId[]; expectedRuntimeMode: RunnerGeneration; expectedRuntimeMetadata: { @@ -85,8 +88,8 @@ export interface EnvironmentFixture { label: string; groups: readonly string[]; driver: "local" | "sandbox"; - provider: "local" | "daytona"; - credential?: "DAYTONA_API_KEY"; + provider: "local" | "daytona" | "exe-dev"; + credential?: "DAYTONA_API_KEY" | "EXE_DEV_SSH_PRIVATE_KEY"; lifecycle: { setup: "instance_managed" | "create_via_api"; probe: "run_context_via_api"; @@ -180,7 +183,7 @@ export interface MatrixJob { executionId: string; suiteId: string; profileId: string; - credentialName: Exclude; + credentialName: Exclude; environmentId: RunnerEnvironmentId; caseId: string; timeoutMinutes: number; diff --git a/ui/src/pages/InstanceExperimentalSettings.test.tsx b/ui/src/pages/InstanceExperimentalSettings.test.tsx index dd69265d56a..0e132d49dbe 100644 --- a/ui/src/pages/InstanceExperimentalSettings.test.tsx +++ b/ui/src/pages/InstanceExperimentalSettings.test.tsx @@ -70,6 +70,7 @@ const PAPERCLIP_RUNNER_TOGGLE_SELECTOR = function defaultExperimentalSettings(): InstanceExperimentalSettingsPayload { return { enableEnvironments: false, + enableExeEnvironments: false, enableNativeRunner: false, enableManagedSandboxOnly: false, enableIsolatedWorkspaces: false, @@ -661,6 +662,13 @@ describe("InstanceExperimentalSettings — Conference Room Chat card (PAP-11233) container.querySelector(SUMMARIES_TOGGLE_SELECTOR)?.getAttribute("aria-checked"), ).toBe("true"); }); + it("enables environment management with the exe.dev experiment", async () => { + await renderPage(); + const toggle = container.querySelector('button[aria-label="Toggle exe.dev environments experimental setting"]'); + await act(async () => { toggle?.click(); }); + await flushReact(); + expect(mockInstanceSettingsApi.updateExperimental).toHaveBeenCalledWith({ enableExeEnvironments: true, enableEnvironments: true }); + }); it("disables Status Cards when disabling Summaries", async () => { currentExperimentalSettings = { diff --git a/ui/src/pages/InstanceExperimentalSettings.tsx b/ui/src/pages/InstanceExperimentalSettings.tsx index c6612681545..c1fb6da6456 100644 --- a/ui/src/pages/InstanceExperimentalSettings.tsx +++ b/ui/src/pages/InstanceExperimentalSettings.tsx @@ -372,6 +372,18 @@ export function InstanceExperimentalSettings() { ariaLabel="Toggle environments experimental setting" /> + toggleMutation.mutate(checked ? { enableExeEnvironments: true, ...(managedKeys.enableEnvironments?.managed ? {} : { enableEnvironments: true }) } : { enableExeEnvironments: false })} + disabled={toggleMutation.isPending || (managedKeys.enableEnvironments?.managed === true && !enableEnvironments)} + settingKey="enableExeEnvironments" + managed={managedKeys.enableExeEnvironments} + ariaLabel="Toggle exe.dev environments experimental setting" + /> +