PaperClipAI/packages/adapter-utils/src/sandbox-callback-bridge.test.ts

import { execFile as execFileCallback, spawn } from "node:child_process";
import { createServer } from "node:http";
import fs, { mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import { afterEach, describe, expect, it, vi } from "vitest";

import { getActiveStepContext, measureStartupStep } from "./acpx-engine/startup-timing.js";
import { prepareCommandManagedRuntime } from "./command-managed-runtime.js";
import {
  authorizeSandboxCallbackBridgeRequestWithRoutes,
  createCommandManagedSandboxCallbackBridgeQueueClient,
  createFileSystemSandboxCallbackBridgeQueueClient,
  createSandboxCallbackBridgeAsset,
  createSandboxCallbackBridgeToken,
  getSandboxBridgeProcessBodyLedgerSource,
  getSandboxCallbackBridgeServerSource,
  HTTP2_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST,
  sandboxCallbackBridgeDirectories,
  syncRemoteTextFileWithHashSkip,
  syncSandboxCallbackBridgeEntrypoint,
  startSandboxCallbackBridgeServer,
  startSandboxCallbackBridgeWorker,
} from "./sandbox-callback-bridge.js";
import type { SandboxCallbackBridgeQueueClient } from "./sandbox-callback-bridge.js";
import { DUPLEX_FRAME_VERSION, encodeDuplexFrame } from "./duplex-frame-codec.js";
import { createHttp2BridgeServer } from "./http2-bridge-server.js";
import type { Http2BridgeForwardRequest, Http2BridgeForwardResult } from "./http2-bridge-server.js";
import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
import type { RuntimeSpanRunner } from "./acpx-engine/startup-timing.js";
import type { RunProcessResult } from "./server-utils.js";

const execFile = promisify(execFileCallback);

describe("sandbox callback bridge", () => {
  const cleanupDirs: string[] = [];
  function queueRuntimeInput(
    runner: ReturnType<typeof createExecRunner>,
    remoteWorkspaceDir: string,
    localWorkspaceDir: string,
    bridgeAsset: Awaited<ReturnType<typeof createSandboxCallbackBridgeAsset>>,
  ): Parameters<typeof prepareCommandManagedRuntime>[0] {
    return {
      runner,
      spec: {
        remoteCwd: remoteWorkspaceDir,
        timeoutMs: 30_000,
      },
      adapterKey: "codex",
      workspaceLocalDir: localWorkspaceDir,
      assets: [
        {
          key: "bridge",
          localDir: bridgeAsset.localDir,
        },
      ],
    };
  }

  function createWorkerWaiter() {
    return async (predicate: () => boolean, timeoutMs: number) => {
      const deadline = Date.now() + timeoutMs;
      while (Date.now() < deadline) {
        if (predicate()) {
          return;
        }
        await new Promise((resolve) => setTimeout(resolve, 5));
      }
      throw new Error("waitFor timed out");
    };
  }

  function virtualQueueReadClient(
    directories: ReturnType<typeof sandboxCallbackBridgeDirectories>,
    requestBodies: Map<string, string>,
  ): Pick<SandboxCallbackBridgeQueueClient, "makeDir" | "makeDirs" | "listJsonFiles" | "readTextFile" | "writeTextFile"> {
    return {
      makeDir: async () => {},
      makeDirs: async () => {},
      listJsonFiles: async (dir) =>
        dir === directories.requestsDir
          ? [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort()
          : [],
      readTextFile: async (remotePath) => {
        const body = requestBodies.get(remotePath);
        if (body === undefined) {
          throw new Error(`missing request ${remotePath}`);
        }
        return body;
      },
      writeTextFile: async () => {},
    };
  }

  const cleanupFns: Array<() => Promise<void>> = [];

  function createExecRunner() {
    return {
      execute: async (input: {
        command: string;
        args?: string[];
        cwd?: string;
        env?: Record<string, string>;
        stdin?: string;
        timeoutMs?: number;
      }): Promise<RunProcessResult> => {
        const startedAt = new Date().toISOString();
        const env = {
          ...process.env,
          ...input.env,
        };
        const command =
          input.command === "sh" ? "/bin/sh" : input.command === "bash" ? "/bin/bash" : input.command;
        const args = [...(input.args ?? [])];
        if (
          input.stdin != null &&
          (input.command === "sh" || input.command === "bash") &&
          (args[0] === "-c" || args[0] === "-lc") &&
          typeof args[1] === "string"
        ) {
          env.PAPERCLIP_TEST_STDIN = input.stdin;
          args[1] = `printf '%s' \"$PAPERCLIP_TEST_STDIN\" | (${args[1]})`;
        }
        try {
          const result = await execFile(command, args, {
            cwd: input.cwd,
            env,
            maxBuffer: 32 * 1024 * 1024,
            timeout: input.timeoutMs,
          });
          return {
            exitCode: 0,
            signal: null,
            timedOut: false,
            stdout: result.stdout,
            stderr: result.stderr,
            pid: null,
            startedAt,
          };
        } catch (error) {
          const err = error as NodeJS.ErrnoException & {
            stdout?: string;
            stderr?: string;
            code?: string | number | null;
            signal?: NodeJS.Signals | null;
            killed?: boolean;
          };
          return {
            exitCode: typeof err.code === "number" ? err.code : null,
            signal: err.signal ?? null,
            timedOut: Boolean(err.killed && input.timeoutMs),
            stdout: err.stdout ?? "",
            stderr: err.stderr ?? "",
            pid: null,
            startedAt,
          };
        }
      },
    };
  }

  // Match the gateway's atomic publication. A visible, growing .json file can
  // be stat'ed while empty and exceed the reader's reserved size on the read.
  function publishBridgeRequest(requestPath: string, body: string): Promise<void> {
    return createFileSystemSandboxCallbackBridgeQueueClient().writeTextFile(requestPath, body);
  }

  async function waitForJsonFile(directory: string, timeoutMs = 2_000): Promise<string> {
    const deadline = Date.now() + timeoutMs;
    while (Date.now() < deadline) {
      const entries = await readdir(directory).catch(() => []);
      const match = entries.find((entry) => entry.endsWith(".json"));
      if (match) return match;
      await new Promise((resolve) => setTimeout(resolve, 10));
    }
    throw new Error(`Timed out waiting for a JSON file in ${directory}.`);
  }

  afterEach(async () => {
    while (cleanupFns.length > 0) {
      const cleanup = cleanupFns.pop();
      if (!cleanup) continue;
      await cleanup().catch(() => undefined);
    }
    while (cleanupDirs.length > 0) {
      const dir = cleanupDirs.pop();
      if (!dir) continue;
      await rm(dir, { recursive: true, force: true }).catch(() => undefined);
    }
  });

  it("round-trips localhost bridge requests over the sandbox queue without forwarding the bridge token", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-runtime-"));
    cleanupDirs.push(rootDir);

    const localWorkspaceDir = path.join(rootDir, "local-workspace");
    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    await mkdir(localWorkspaceDir, { recursive: true });
    await mkdir(remoteWorkspaceDir, { recursive: true });
    await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge test\n", "utf8");

    const runner = createExecRunner();

    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);

    const prepared = await prepareCommandManagedRuntime(queueRuntimeInput(runner, remoteWorkspaceDir, localWorkspaceDir, bridgeAsset));

    const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const bridgeToken = createSandboxCallbackBridgeToken();
    const seenRequests: Array<{
      method: string;
      path: string;
      query: string;
      headers: Record<string, string>;
      body: string | Buffer;
    }> = [];

    const worker = await startSandboxCallbackBridgeWorker({
      client: createFileSystemSandboxCallbackBridgeQueueClient(),
      queueDir,
      authorizeRequest: async (request) =>
        ["/api/agents/me", "/runtime-tools/github/credentials"].includes(request.path) ? null
          : request.path.endsWith("/agent-commentary") ? authorizeSandboxCallbackBridgeRequestWithRoutes(request)
          : `Route not allowed: ${request.method} ${request.path}`,
      handleRequest: async (request) => {
        seenRequests.push({
          method: request.method,
          path: request.path,
          query: request.query,
          headers: request.headers,
          body: request.body,
        });
        return {
          status: 200,
          headers: {
            "content-type": "application/json",
            etag: '"bridge-rev-1"',
            "last-modified": "Tue, 01 Apr 2025 00:00:00 GMT",
          },
          body: JSON.stringify({
            ok: true,
            method: request.method,
            path: request.path,
          }),
        };
      },
    });
    cleanupFns.push(async () => {
      await worker.stop();
    });

    const bridge = await startSandboxCallbackBridgeServer({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: prepared.assetDirs.bridge,
      queueDir,
      bridgeToken,
      timeoutMs: 30_000,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    const okResponse = await fetch(`${bridge.baseUrl}/api/agents/me?view=compact`, {
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        accept: "application/json",
        "if-none-match": '"client-cache-key"',
        "x-paperclip-run-id": "run-bridge-1",
        "x-bridge-debug": "drop-me",
      },
    });
    expect(okResponse.status).toBe(200);
    expect(okResponse.headers.get("content-type")).toContain("application/json");
    expect(okResponse.headers.get("etag")).toBe('"bridge-rev-1"');
    expect(okResponse.headers.get("last-modified")).toBe("Tue, 01 Apr 2025 00:00:00 GMT");
    await expect(okResponse.json()).resolves.toMatchObject({
      ok: true,
      method: "GET",
      path: "/api/agents/me",
    });

    const deniedResponse = await fetch(`${bridge.baseUrl}/api/issues/issue-1`, {
      method: "PATCH",
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "content-type": "application/json",
      },
      body: JSON.stringify({ status: "in_progress" }),
    });
    expect(deniedResponse.status).toBe(403);
    await expect(deniedResponse.json()).resolves.toMatchObject({
      error: "Route not allowed: PATCH /api/issues/issue-1",
    });

    const unauthorizedResponse = await fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: {
        authorization: "Bearer wrong-token",
      },
    });
    expect(unauthorizedResponse.status).toBe(401);
    await expect(unauthorizedResponse.json()).resolves.toMatchObject({
      error: "Invalid bridge token.",
    });

    expect(seenRequests).toHaveLength(1);
    expect(seenRequests[0]).toMatchObject({
      method: "GET",
      path: "/api/agents/me",
      query: "?view=compact",
      body: "",
      headers: {
        accept: "application/json",
        "if-none-match": '"client-cache-key"',
      },
    });
    expect(seenRequests[0]?.headers.authorization).toBeUndefined();
    expect(seenRequests[0]?.headers["x-paperclip-run-id"]).toBeUndefined();

    // The public Paperclip API documents the "X-API-Key" header, so an agent in
    // the sandbox sends that header before it sends "Authorization: Bearer".
    // The gateway accepts the same per-run token through either header.
    const apiKeyResponse = await fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: {
        "x-api-key": bridgeToken,
        accept: "application/json",
      },
    });
    expect(apiKeyResponse.status).toBe(200);
    await expect(apiKeyResponse.json()).resolves.toMatchObject({
      ok: true,
      method: "GET",
      path: "/api/agents/me",
    });

    // A wrong token in the same header still fails closed.
    const apiKeyDeniedResponse = await fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: {
        "x-api-key": "wrong-token",
      },
    });
    expect(apiKeyDeniedResponse.status).toBe(401);
    await expect(apiKeyDeniedResponse.json()).resolves.toMatchObject({
      error: "Invalid bridge token.",
    });

    // The header allowlist strips the credential header, so the token never
    // leaves the sandbox with the forwarded request.
    expect(seenRequests).toHaveLength(2);
    expect(seenRequests[1]).toMatchObject({
      method: "GET",
      path: "/api/agents/me",
    });
    expect(seenRequests[1]?.headers["x-api-key"]).toBeUndefined();

    const githubResponse = await fetch(`${bridge.baseUrl}/runtime-tools/github/credentials`, {
      method: "POST",
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "content-type": "application/json",
        "x-paperclip-github-capability": "test-run-scoped-capability",
      },
      body: "{}",
    });
    expect(githubResponse.status).toBe(200);
    await githubResponse.arrayBuffer();
    expect(seenRequests[2]).toMatchObject({
      method: "POST", path: "/runtime-tools/github/credentials", body: "{}",
      headers: { "x-paperclip-github-capability": "test-run-scoped-capability" },
    });
    expect(seenRequests[2]?.headers.authorization).toBeUndefined();

    const feedbackBody = JSON.stringify({ kind: "complaint", body: "😭".repeat(5000), idempotencyKey: "bridge-feedback" });
    const feedbackResponse = await fetch(`${bridge.baseUrl}/api/companies/company-1/agent-commentary`, {
      method: "POST", headers: { authorization: `Bearer ${bridgeToken}`, "content-type": "application/json" }, body: feedbackBody,
    });
    expect(feedbackResponse.status).toBe(200);
    await feedbackResponse.arrayBuffer();
    expect(seenRequests).toHaveLength(4);
    expect(seenRequests[3]).toMatchObject({ method: "POST", path: "/api/companies/company-1/agent-commentary", body: feedbackBody });
    expect(seenRequests[3]?.headers.authorization).toBeUndefined();

  });

  it("publishes complete fixture envelopes before queue readers can reserve their size", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-publication-"));
    cleanupDirs.push(rootDir);
    const client = createFileSystemSandboxCallbackBridgeQueueClient();
    const requestPath = path.join(rootDir, "fixture.json");
    const payload = JSON.stringify({ id: "fixture", method: "GET", path: "/api/openapi.json", body: "" });
    const listingsDuringWrite: string[][] = [];
    const write = fs.writeFile.bind(fs);
    // Force a poll while the writer has created its file but not filled it.
    // A direct .json write would become visible and reserve zero bytes here.
    const spy = vi.spyOn(fs, "writeFile").mockImplementation(async (file, body, options) => {
      if (typeof file !== "string" || path.dirname(file) !== rootDir) return write(file, body, options);
      await write(file, "", options);
      const visible = await client.listJsonFiles(rootDir);
      listingsDuringWrite.push(visible);
      const reserved = visible.includes("fixture.json") ? await client.fileSize!(requestPath) : null;
      await write(file, body, options);
      if (reserved !== null) await client.readTextFile(requestPath, reserved);
    });
    try {
      await publishBridgeRequest(requestPath, payload);
      expect(listingsDuringWrite).toEqual([[]]);
      expect(await client.listJsonFiles(rootDir)).toEqual(["fixture.json"]);
      const size = await client.fileSize!(requestPath);
      expect(size).toBe(Buffer.byteLength(payload));
      await expect(client.readTextFile(requestPath, size)).resolves.toBe(payload);
    } finally {
      spy.mockRestore();
    }
  });

  it("keeps the bounded reader fail-closed when a visible request grows after its size is reserved", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-growing-request-"));
    cleanupDirs.push(rootDir);
    const queueDir = path.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const client = createFileSystemSandboxCallbackBridgeQueueClient();
    const requestPath = path.join(directories.requestsDir, "growing.json");
    await mkdir(directories.requestsDir, { recursive: true });
    // Deliberately violate the publication contract to reproduce the old
    // fixture interleaving, without timing assumptions or a weakened limit.
    await writeFile(requestPath, "");
    const payload = JSON.stringify({ id: "growing", method: "GET", path: "/api/openapi.json", body: "" });
    let reserved: number | undefined;
    const handleRequest = vi.fn(async () => ({ status: 200, body: "unexpected" }));
    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
    try {
      const worker = await startSandboxCallbackBridgeWorker({
        queueDir, handleRequest,
        client: { ...client, fileSize: async (file) => {
          const size = await client.fileSize!(file);
          if (file === requestPath && reserved === undefined) {
            reserved = size;
            await writeFile(requestPath, payload);
          }
          return size;
        } },
      });
      cleanupFns.push(() => worker.stop());
      const responseFile = await waitForJsonFile(directories.responsesDir);
      const response = JSON.parse(await readFile(path.join(directories.responsesDir, responseFile), "utf8"));
      expect(reserved).toBe(0);
      expect(response.status).toBe(503);
      expect(JSON.parse(response.body)).toEqual({
        error: "Sandbox callback bridge worker failed: Bridge envelope exceeded the configured size limit.",
      });
      expect(warn).toHaveBeenCalledWith(expect.stringContaining("Bridge envelope exceeded the configured size limit."));
      expect(handleRequest).not.toHaveBeenCalled();
      await worker.stop();
    } finally {
      warn.mockRestore();
    }
  });

  it("serves schema discovery over the queue and denies schema mutations and lookalikes", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-schema-"));
    cleanupDirs.push(rootDir);
    const queueDir = path.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const schema = { openapi: "3.1.0", paths: {} };
    const forwarded: string[] = [];
    const worker = await startSandboxCallbackBridgeWorker({
      client: createFileSystemSandboxCallbackBridgeQueueClient(), queueDir,
      handleRequest: async (request) => {
        forwarded.push(`${request.method} ${request.path}`);
        return { status: 200, body: JSON.stringify(schema) };
      },
    });
    cleanupFns.push(() => worker.stop());
    const requests = [
      { method: "GET", path: "/api/openapi.json" },
      { method: "POST", path: "/api/openapi.json" },
      { method: "PATCH", path: "/api/openapi.json" },
      { method: "DELETE", path: "/api/openapi.json" },
      { method: "GET", path: "/api/openapi.json/extra" },
      { method: "GET", path: "/api/openapiXjson" },
      { method: "GET", path: "/api/secrets" },
    ];
    for (const [index, request] of requests.entries()) {
      await publishBridgeRequest(path.join(directories.requestsDir, `schema-${index}.json`), JSON.stringify({
        id: `schema-${index}`, ...request, query: "", headers: {}, body: "", createdAt: new Date().toISOString(),
      }));
    }
    await vi.waitFor(async () => {
      expect(await createFileSystemSandboxCallbackBridgeQueueClient().listJsonFiles(directories.responsesDir))
        .toEqual(requests.map((_, index) => `schema-${index}.json`));
    }, { timeout: 5_000 });
    for (const [index] of requests.entries()) {
      const response = JSON.parse(await readFile(path.join(directories.responsesDir, `schema-${index}.json`), "utf8"));
      expect(response.status).toBe(index === 0 ? 200 : 403);
      if (index === 0) expect(JSON.parse(response.body)).toEqual(schema);
    }
    expect(forwarded).toEqual(["GET /api/openapi.json"]);
  });

  it("denies non-allowlisted requests by default", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-default-policy-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    let handled = 0;

    const worker = await startSandboxCallbackBridgeWorker({
      client: createFileSystemSandboxCallbackBridgeQueueClient(),
      queueDir,
      handleRequest: async () => {
        handled += 1;
        return {
          status: 200,
          body: "should not happen",
        };
      },
    });

    await publishBridgeRequest(
      path.posix.join(directories.requestsDir, "req-1.json"),
      `${JSON.stringify({
        id: "req-1",
        method: "DELETE",
        path: "/api/secrets",
        query: "",
        headers: {},
        body: "",
        createdAt: new Date().toISOString(),
      })}\n`,
    );

    await worker.stop({ drainTimeoutMs: 1_000 });

    const response = JSON.parse(
      await readFile(path.posix.join(directories.responsesDir, "req-1.json"), "utf8"),
    ) as { status: number; body: string };
    expect(handled).toBe(0);
    expect(response.status).toBe(403);
    expect(JSON.parse(response.body)).toEqual({
      error: "Route not allowed: DELETE /api/secrets",
    });
  });

  it("drains already-queued requests on stop", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-drain-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const processed: string[] = [];

    const worker = await startSandboxCallbackBridgeWorker({
      client: createFileSystemSandboxCallbackBridgeQueueClient(),
      queueDir,
      authorizeRequest: async () => null,
      handleRequest: async (request) => {
        processed.push(request.id);
        await new Promise((resolve) => setTimeout(resolve, 25));
        return {
          status: 200,
          body: request.id,
        };
      },
    });

    await publishBridgeRequest(
      path.posix.join(directories.requestsDir, "req-a.json"),
      `${JSON.stringify({
        id: "req-a",
        method: "GET",
        path: "/api/agents/me",
        query: "",
        headers: {},
        body: "",
        createdAt: new Date().toISOString(),
      })}\n`,
    );
    await publishBridgeRequest(
      path.posix.join(directories.requestsDir, "req-b.json"),
      `${JSON.stringify({
        id: "req-b",
        method: "GET",
        path: "/api/agents/me",
        query: "",
        headers: {},
        body: "",
        createdAt: new Date().toISOString(),
      })}\n`,
    );

    await worker.stop({ drainTimeoutMs: 1_000 });

    expect(processed).toEqual(["req-a", "req-b"]);
    await expect(readFile(path.posix.join(directories.responsesDir, "req-a.json"), "utf8")).resolves.toContain("\"req-a\"");
    await expect(readFile(path.posix.join(directories.responsesDir, "req-b.json"), "utf8")).resolves.toContain("\"req-b\"");
  });

  it("writes fast 503 responses for queued requests that miss the drain deadline", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-drain-timeout-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const processed: string[] = [];
    let signalStarted!: () => void;
    const started = new Promise<void>(resolve => { signalStarted = resolve; });

    const worker = await startSandboxCallbackBridgeWorker({
      client: createFileSystemSandboxCallbackBridgeQueueClient(),
      queueDir,
      authorizeRequest: async () => null,
      handleRequest: async (request) => {
        processed.push(request.id);
        signalStarted();
        await new Promise((resolve) => setTimeout(resolve, 100));
        return {
          status: 200,
          body: request.id,
        };
      },
    });

    await publishBridgeRequest(
      path.posix.join(directories.requestsDir, "req-a.json"),
      `${JSON.stringify({
        id: "req-a",
        method: "GET",
        path: "/api/agents/me",
        query: "",
        headers: {},
        body: "",
        createdAt: new Date().toISOString(),
      })}\n`,
    );
    await publishBridgeRequest(
      path.posix.join(directories.requestsDir, "req-b.json"),
      `${JSON.stringify({
        id: "req-b",
        method: "GET",
        path: "/api/agents/me",
        query: "",
        headers: {},
        body: "",
        createdAt: new Date().toISOString(),
      })}\n`,
    );

    // Begin the short drain deadline only after the first handler has started.
    // A fixed 250ms polling window can expire during a loaded test run.
    await started;

    await worker.stop({ drainTimeoutMs: 10 });

    expect(processed).toEqual(["req-a"]);
    await expect(readFile(path.posix.join(directories.responsesDir, "req-a.json"), "utf8")).resolves.toContain("\"req-a\"");
    await expect(readFile(path.posix.join(directories.responsesDir, "req-b.json"), "utf8")).resolves.toContain(
      "Bridge worker stopped before request could be handled.",
    );
  });

  it("handles SSH queue polling failures without emitting an unhandled rejection", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-ssh-failure-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const unhandled: unknown[] = [];
    const onUnhandledRejection = (reason: unknown) => {
      unhandled.push(reason);
    };
    process.on("unhandledRejection", onUnhandledRejection);

    try {
      const worker = await startSandboxCallbackBridgeWorker({
        client: {
          makeDir: async () => {},
          makeDirs: async () => {},
          listJsonFiles: async () => {
            throw new Error(
              "list /remote/.paperclip-runtime/gemini/paperclip-bridge/queue/requests failed with exit code 255: kex_exchange_identification: read: Connection reset by peer",
            );
          },
          readTextFile: async () => {
            throw new Error("unexpected readTextFile");
          },
          writeTextFile: async () => {
            throw new Error("unexpected writeTextFile");
          },
          rename: async () => {
            throw new Error("unexpected rename");
          },
          remove: async () => {},
        },
        queueDir,
        authorizeRequest: async () => null,
        handleRequest: async () => ({
          status: 200,
          body: "ok",
        }),
      });

      await new Promise((resolve) => setTimeout(resolve, 50));
      await worker.stop();
      expect(unhandled).toEqual([]);
    } finally {
      process.off("unhandledRejection", onUnhandledRejection);
    }
  });

  it("recovers from transient queue polling failures and keeps relaying requests", async () => {
    // A single reset or slow sandbox exec used to unwind the poll loop into its
    // terminal catch, killing the relay for the rest of the run. The loop must
    // instead back off, retry, and still deliver requests queued after the
    // failure window.
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-transient-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const baseClient = createFileSystemSandboxCallbackBridgeQueueClient();
    let listCalls = 0;
    const client: SandboxCallbackBridgeQueueClient = {
      ...baseClient,
      listJsonFiles: async (dirPath: string) => {
        listCalls += 1;
        if (listCalls <= 3) {
          throw new Error("list requests failed: kex_exchange_identification: read: Connection reset by peer");
        }
        return baseClient.listJsonFiles(dirPath);
      },
    };

    const seenPaths: string[] = [];
    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      authorizeRequest: async () => null,
      handleRequest: async (request) => {
        seenPaths.push(request.path);
        return {
          status: 200,
          headers: { "content-type": "application/json" },
          body: JSON.stringify({ ok: true }),
        };
      },
    });
    cleanupFns.push(async () => {
      await worker.stop();
    });

    const requestId = "transient-recovery-1";
    await publishBridgeRequest(
      path.join(directories.requestsDir, `${requestId}.json`),
      JSON.stringify({
        id: requestId,
        method: "GET",
        path: "/api/agents/me",
        query: "",
        headers: {},
        body: "",
      }),
    );

    const responseFile = await waitForJsonFile(directories.responsesDir, 10_000);
    const raw = await readFile(path.join(directories.responsesDir, responseFile), "utf8");
    expect(JSON.parse(raw)).toMatchObject({ id: requestId, status: 200 });
    expect(seenPaths).toEqual(["/api/agents/me"]);
    expect(listCalls).toBeGreaterThan(3);
  });

  it("keeps the queue-directory setup on the startup step but resets the poll loop store", async () => {
    // The worker starts inside the measured `bridge.paperclip` step. Its awaited
    // queue-directory setup is startup work, so a `makeDir` `sandbox.exec` span
    // must keep the active step and its `criticalPath` flag. The long-lived poll
    // loop runs run-time execs for the whole run, so a loop `sandbox.exec` span
    // must open unparented with no stale flag. This test reads the active step in
    // both places and proves the boundary sits at the loop, not the whole worker.
    let setupStep: ReturnType<typeof getActiveStepContext> | "unset" = "unset";
    let loopStep: ReturnType<typeof getActiveStepContext> | "unset" = "unset";
    let resolveFirstPoll: () => void = () => {};
    const firstPoll = new Promise<void>((resolve) => {
      resolveFirstPoll = resolve;
    });

    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-step-store-"));
    cleanupDirs.push(rootDir);
    const queueDir = path.posix.join(rootDir, "queue");

    const worker = await measureStartupStep(
      {},
      () => 0,
      "bridge.paperclip",
      () =>
        startSandboxCallbackBridgeWorker({
          client: {
            makeDir: async () => {
              setupStep = getActiveStepContext();
            },
            makeDirs: async () => {
              setupStep = getActiveStepContext();
            },
            listJsonFiles: async () => {
              loopStep = getActiveStepContext();
              resolveFirstPoll();
              return [];
            },
            readTextFile: async () => {
              throw new Error("unexpected readTextFile");
            },
            writeTextFile: async () => {
              throw new Error("unexpected writeTextFile");
            },
            rename: async () => {
              throw new Error("unexpected rename");
            },
            remove: async () => {},
          },
          queueDir,
          authorizeRequest: async () => null,
          handleRequest: async () => ({ status: 200, body: "ok" }),
        }),
      { criticalPath: false },
    );

    await firstPoll;
    await worker.stop();

    // The setup ran on the active step, so its exec span parents to the step.
    expect(setupStep).not.toBe("unset");
    expect(setupStep).not.toBeNull();
    expect((setupStep as { criticalPath?: boolean }).criticalPath).toBe(false);

    // The loop ran outside that store, so its exec span opens unparented with no
    // stale `criticalPath` flag.
    expect(loopStep).toBeNull();
  });

  it("test_paperclip_loop_exec_parents_to_run_context", async () => {
    // The worker starts inside the measured `bridge.paperclip` step. Its awaited
    // queue-directory setup is startup work and keeps the active step. The poll
    // loop shell stays outside that store. But a per-request unit of work is
    // run-time work, so the worker runs each request under the current-run
    // parent context. A request `sandbox.exec` span then parents to the live run
    // span, not to the ended startup step. This test drives the worker with a
    // `getRuntimeParentContext` that returns a known token, queues one request,
    // and proves the request work reads that token from the active step store.
    const runParentToken = { marker: "run-parent-token" };
    let setupStep: ReturnType<typeof getActiveStepContext> | "unset" = "unset";
    let requestStep: ReturnType<typeof getActiveStepContext> | "unset" = "unset";
    let served = false;
    let resolveServed: () => void = () => {};
    const requestServed = new Promise<void>((resolve) => {
      resolveServed = resolve;
    });

    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-run-parent-"));
    cleanupDirs.push(rootDir);
    const queueDir = path.posix.join(rootDir, "queue");

    const worker = await measureStartupStep(
      {},
      () => 0,
      "bridge.paperclip",
      () =>
        startSandboxCallbackBridgeWorker({
          client: {
            makeDir: async () => {
              setupStep = getActiveStepContext();
            },
            makeDirs: async () => {
              setupStep = getActiveStepContext();
            },
            // Return one request on the first poll, then nothing.
            listJsonFiles: async () => (served ? [] : ["000000000001.json"]),
            readTextFile: async () =>
              JSON.stringify({ id: "req-1", method: "GET", path: "/", query: "", headers: {}, body: "" }),
            writeTextFile: async () => {},
            rename: async () => {},
            remove: async () => {},
          },
          queueDir,
          authorizeRequest: async () => null,
          handleRequest: async () => {
            requestStep = getActiveStepContext();
            served = true;
            resolveServed();
            return { status: 200, body: "ok" };
          },
          getRuntimeParentContext: () => runParentToken,
        }),
      { criticalPath: false },
    );

    await requestServed;
    await worker.stop();

    // The setup ran on the active step, so its exec span parents to the step.
    expect(setupStep).not.toBe("unset");
    expect(setupStep).not.toBeNull();

    // The request work ran under the run parent context. Its exec span parents
    // to the run token, not to the ended startup step, and it carries no
    // startup `criticalPath` flag.
    expect(requestStep).not.toBe("unset");
    expect(requestStep).not.toBeNull();
    expect((requestStep as { parentContext?: unknown }).parentContext).toBe(runParentToken);
    expect((requestStep as { criticalPath?: boolean }).criticalPath).toBe(false);
  });

  it("wraps each request in a sandbox.callbackBridge.relayRequest span", async () => {
    // With a span runner injected, the worker wraps each request in one
    // `sandbox.callbackBridge.relayRequest` span, so the request's read, write,
    // and remove execs group under one named span. This test drives the worker
    // with a recording runner and proves it opens the wrapper span around the
    // request work.
    const wrapped: string[] = [];
    let served = false;
    let resolveServed: () => void = () => {};
    const requestServed = new Promise<void>((resolve) => {
      resolveServed = resolve;
    });

    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-relay-span-"));
    cleanupDirs.push(rootDir);
    const queueDir = path.posix.join(rootDir, "queue");

    const worker = await startSandboxCallbackBridgeWorker({
      client: {
        makeDir: async () => {},
        makeDirs: async () => {},
        listJsonFiles: async () => (served ? [] : ["000000000001.json"]),
        readTextFile: async () =>
          JSON.stringify({ id: "req-1", method: "GET", path: "/", query: "", headers: {}, body: "" }),
        writeTextFile: async () => {},
        rename: async () => {},
        remove: async () => {},
      },
      queueDir,
      authorizeRequest: async () => null,
      handleRequest: async () => {
        served = true;
        resolveServed();
        return { status: 200, body: "ok" };
      },
      // Record each wrapper span name, then run the wrapped work.
      runtimeSpan: async (name, work) => {
        wrapped.push(name);
        return work();
      },
    });

    await requestServed;
    await worker.stop();

    expect(wrapped).toContain("sandbox.callbackBridge.relayRequest");
  });

  it("test_paperclip_loop_exec_stays_unparented_without_getter", async () => {
    // With no `getRuntimeParentContext`, a request runs with an empty active
    // step store, exactly like the earlier `runWithoutActiveStep` behavior. So a
    // request `sandbox.exec` span opens unparented with no stale startup flag.
    let requestStep: ReturnType<typeof getActiveStepContext> | "unset" = "unset";
    let served = false;
    let resolveServed: () => void = () => {};
    const requestServed = new Promise<void>((resolve) => {
      resolveServed = resolve;
    });

    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-no-getter-"));
    cleanupDirs.push(rootDir);
    const queueDir = path.posix.join(rootDir, "queue");

    const worker = await measureStartupStep(
      {},
      () => 0,
      "bridge.paperclip",
      () =>
        startSandboxCallbackBridgeWorker({
          client: {
            makeDir: async () => {},
            makeDirs: async () => {},
            listJsonFiles: async () => (served ? [] : ["000000000001.json"]),
            readTextFile: async () =>
              JSON.stringify({ id: "req-1", method: "GET", path: "/", query: "", headers: {}, body: "" }),
            writeTextFile: async () => {},
            rename: async () => {},
            remove: async () => {},
          },
          queueDir,
          authorizeRequest: async () => null,
          handleRequest: async () => {
            requestStep = getActiveStepContext();
            served = true;
            resolveServed();
            return { status: 200, body: "ok" };
          },
        }),
      { criticalPath: false },
    );

    await requestServed;
    await worker.stop();

    expect(requestStep).toBeNull();
  });

  it("serializes remote response writes so stop does not recreate a late orphaned response", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-response-lock-"));
    cleanupDirs.push(rootDir);

    const localWorkspaceDir = path.join(rootDir, "local-workspace");
    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    await mkdir(localWorkspaceDir, { recursive: true });
    await mkdir(remoteWorkspaceDir, { recursive: true });
    await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge response lock test\n", "utf8");

    const runner = createExecRunner();
    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const prepared = await prepareCommandManagedRuntime(queueRuntimeInput(runner, remoteWorkspaceDir, localWorkspaceDir, bridgeAsset));

    const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const bridgeToken = createSandboxCallbackBridgeToken();
    const seenRequestIds: string[] = [];

    const worker = await startSandboxCallbackBridgeWorker({
      client: createCommandManagedSandboxCallbackBridgeQueueClient({
        runner,
        remoteCwd: remoteWorkspaceDir,
        timeoutMs: 30_000,
      }),
      queueDir,
      authorizeRequest: async () => null,
      handleRequest: async (request) => {
        seenRequestIds.push(request.id);
        await new Promise((resolve) => setTimeout(resolve, 250));
        return {
          status: 200,
          headers: { "content-type": "application/json" },
          body: JSON.stringify({ ok: true, id: request.id }),
        };
      },
    });
    cleanupFns.push(async () => {
      await worker.stop();
    });

    const bridge = await startSandboxCallbackBridgeServer({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: prepared.assetDirs.bridge,
      queueDir,
      bridgeToken,
      timeoutMs: 30_000,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    const responsePromise = fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: {
        authorization: `Bearer ${bridgeToken}`,
      },
    });

    for (let attempt = 0; attempt < 50 && seenRequestIds.length === 0; attempt += 1) {
      await new Promise((resolve) => setTimeout(resolve, 5));
    }

    expect(seenRequestIds).toHaveLength(1);
    await worker.stop({ drainTimeoutMs: 10 });

    const response = await responsePromise;
    expect(response.status).toBe(503);
    await expect(response.json()).resolves.toEqual({
      error: "Bridge worker stopped before request could be handled.",
    });

    await new Promise((resolve) => setTimeout(resolve, 300));

    await expect(readdir(directories.responsesDir)).resolves.toEqual([]);
    await expect(
      readdir(directories.responsesDir).then((entries) =>
        entries.filter((entry) => entry.endsWith(".tmp") || entry.includes(".paperclip-write.lock")),
      ),
    ).resolves.toEqual([]);
  });

  it("rejects non-JSON request bodies and full queues at the bridge server", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-server-guards-"));
    cleanupDirs.push(rootDir);

    const localWorkspaceDir = path.join(rootDir, "local-workspace");
    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    await mkdir(localWorkspaceDir, { recursive: true });
    await mkdir(remoteWorkspaceDir, { recursive: true });
    await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge guard test\n", "utf8");

    const runner = createExecRunner();

    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const prepared = await prepareCommandManagedRuntime(queueRuntimeInput(runner, remoteWorkspaceDir, localWorkspaceDir, bridgeAsset));

    const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const bridgeToken = createSandboxCallbackBridgeToken();

    const bridge = await startSandboxCallbackBridgeServer({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: prepared.assetDirs.bridge,
      queueDir,
      bridgeToken,
      timeoutMs: 30_000,
      maxQueueDepth: 1,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    await publishBridgeRequest(
      path.posix.join(directories.requestsDir, "existing.json"),
      `${JSON.stringify({
        id: "existing",
        method: "GET",
        path: "/api/agents/me",
        query: "",
        headers: {},
        body: "",
        createdAt: new Date().toISOString(),
      })}\n`,
    );

    const queueFullResponse = await fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: {
        authorization: `Bearer ${bridgeToken}`,
      },
    });
    expect(queueFullResponse.status).toBe(503);
    await expect(queueFullResponse.json()).resolves.toEqual({
      error: "Bridge request queue is full.",
    });

    await rm(path.posix.join(directories.requestsDir, "existing.json"), { force: true });

    const nonJsonResponse = await fetch(`${bridge.baseUrl}/api/issues/issue-1/comments`, {
      method: "POST",
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "content-type": "text/plain",
      },
      body: "not json",
    });
    expect(nonJsonResponse.status).toBe(415);
    await expect(nonJsonResponse.json()).resolves.toEqual({
      error: "Bridge only accepts JSON request bodies.",
    });

    // The queue transport keeps its 415 gate for the attachment upload path
    // too: it carries a string envelope only, so it never admits a binary body.
    const attachmentOctetStreamResponse = await fetch(
      `${bridge.baseUrl}/api/companies/co-1/issues/issue-1/attachments`,
      {
        method: "POST",
        headers: {
          authorization: `Bearer ${bridgeToken}`,
          "content-type": "application/octet-stream",
        },
        body: Buffer.from([0x50, 0x4b, 0x03, 0x04]),
      },
    );
    expect(attachmentOctetStreamResponse.status).toBe(415);
    await expect(attachmentOctetStreamResponse.json()).resolves.toEqual({
      error: "Bridge only accepts JSON request bodies.",
    });
  });

  it("returns a 502 when the host response times out", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-timeout-"));
    cleanupDirs.push(rootDir);

    const localWorkspaceDir = path.join(rootDir, "local-workspace");
    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    await mkdir(localWorkspaceDir, { recursive: true });
    await mkdir(remoteWorkspaceDir, { recursive: true });
    await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge timeout test\n", "utf8");

    const runner = createExecRunner();
    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const prepared = await prepareCommandManagedRuntime(queueRuntimeInput(runner, remoteWorkspaceDir, localWorkspaceDir, bridgeAsset));

    const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
    const bridgeToken = createSandboxCallbackBridgeToken();
    const bridge = await startSandboxCallbackBridgeServer({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: prepared.assetDirs.bridge,
      queueDir,
      bridgeToken,
      timeoutMs: 30_000,
      pollIntervalMs: 10,
      responseTimeoutMs: 75,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    const response = await fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: {
        authorization: `Bearer ${bridgeToken}`,
      },
    });

    expect(response.status).toBe(502);
    await expect(response.json()).resolves.toEqual({
      error: "Timed out waiting for host bridge response.",
    });
  });

  it("returns a 502 for malformed host response files", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-malformed-response-"));
    cleanupDirs.push(rootDir);

    const localWorkspaceDir = path.join(rootDir, "local-workspace");
    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    await mkdir(localWorkspaceDir, { recursive: true });
    await mkdir(remoteWorkspaceDir, { recursive: true });
    await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge malformed response test\n", "utf8");

    const runner = createExecRunner();
    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const prepared = await prepareCommandManagedRuntime(queueRuntimeInput(runner, remoteWorkspaceDir, localWorkspaceDir, bridgeAsset));

    const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const bridgeToken = createSandboxCallbackBridgeToken();
    const bridge = await startSandboxCallbackBridgeServer({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: prepared.assetDirs.bridge,
      queueDir,
      bridgeToken,
      timeoutMs: 30_000,
      pollIntervalMs: 10,
      responseTimeoutMs: 1_000,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    const responsePromise = fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: {
        authorization: `Bearer ${bridgeToken}`,
      },
    });

    const requestFile = await waitForJsonFile(directories.requestsDir);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);
    // Publish the complete malformed envelope atomically, like the host writer.
    // A direct write can instead exercise the reader's file-size race guard.
    await writeFile(
      `${responsePath}.tmp`,
      '{"status":200,"headers":{"content-type":"application/json"},"body"',
      "utf8",
    );
    await fs.rename(`${responsePath}.tmp`, responsePath);

    const response = await responsePromise;
    expect(response.status).toBe(502);
    await expect(response.json()).resolves.toMatchObject({
      error: expect.stringMatching(/JSON|Unexpected|Unterminated/i),
    });
  });

  it("reuses an already-uploaded bridge entrypoint when the remote file hash matches", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-sync-"));
    cleanupDirs.push(rootDir);

    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    const remoteAssetDir = path.posix.join(
      remoteWorkspaceDir,
      ".paperclip-runtime",
      "codex",
      "paperclip-bridge",
      "server",
    );
    await mkdir(remoteWorkspaceDir, { recursive: true });

    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const originalSource = await readFile(bridgeAsset.entrypoint, "utf8");
    const expandedSource = `${originalSource}\n// bridge payload padding\n`;
    await writeFile(bridgeAsset.entrypoint, expandedSource, "utf8");

    const runner = createExecRunner();

    const first = await syncSandboxCallbackBridgeEntrypoint({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: remoteAssetDir,
      bridgeAsset,
      timeoutMs: 30_000,
    });
    const second = await syncSandboxCallbackBridgeEntrypoint({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: remoteAssetDir,
      bridgeAsset,
      timeoutMs: 30_000,
    });

    expect(first.uploaded).toBe(true);
    expect(second.uploaded).toBe(false);
    await expect(readFile(path.posix.join(remoteAssetDir, "paperclip-bridge-server.mjs"), "utf8")).resolves.toBe(expandedSource);
    await expect(
      readdir(remoteAssetDir).then((entries) =>
        entries.filter(
          (entry) =>
            entry.endsWith(".paperclip-upload.b64") ||
            entry.endsWith(".partial") ||
            entry === ".paperclip-bridge-upload.lock",
        ),
      ),
    ).resolves.toEqual([]);
  });

  it("rejects a corrupted bridge entrypoint upload without committing a torn remote file", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-sync-corrupt-"));
    cleanupDirs.push(rootDir);

    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    const remoteAssetDir = path.posix.join(
      remoteWorkspaceDir,
      ".paperclip-runtime",
      "codex",
      "paperclip-bridge",
      "server",
    );
    await mkdir(remoteWorkspaceDir, { recursive: true });

    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const runner = {
      execute: async (input: {
        command: string;
        args?: string[];
        cwd?: string;
        env?: Record<string, string>;
        stdin?: string;
        timeoutMs?: number;
      }) =>
        await createExecRunner().execute({
          ...input,
          stdin: input.stdin != null ? "" : input.stdin,
        }),
    };

    await expect(
      syncSandboxCallbackBridgeEntrypoint({
        runner,
        remoteCwd: remoteWorkspaceDir,
        assetRemoteDir: remoteAssetDir,
        bridgeAsset,
        timeoutMs: 30_000,
      }),
    ).rejects.toThrow(/sha mismatch/i);

    await expect(readFile(path.posix.join(remoteAssetDir, "paperclip-bridge-server.mjs"), "utf8")).rejects.toThrow();
    await expect(
      readdir(remoteAssetDir).then((entries) =>
        entries.filter(
          (entry) =>
            entry.endsWith(".paperclip-upload.b64") ||
            entry.endsWith(".partial") ||
            entry === ".paperclip-bridge-upload.lock",
        ),
      ),
    ).resolves.toEqual([]);
  });

  // The process-session remote script is a static, Paperclip-authored `.mjs`
  // written into the sandbox on every bridge start. `syncRemoteTextFileWithHashSkip`
  // (which now backs that write, mirroring the bridge-entrypoint sha256 gate)
  // content-hash-skips it so a warm start where the remote script already matches
  // costs ZERO write execs instead of the prior ~3 (prepare/append/finalize base64
  // upload).
  it("test_process_session_script_skipped_when_remote_hash_matches: warm start with a matching remote hash writes 0 execs", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-hashskip-warm-"));
    cleanupDirs.push(rootDir);
    const remoteDir = path.join(rootDir, "runtime", "codex", "process-sessions");
    const remotePath = path.posix.join(remoteDir, "paperclip-process-session-remote.mjs");
    const lockDir = path.posix.join(remoteDir, ".paperclip-process-session-script.lock");
    const body = "console.log('process session remote script v1');\n";

    let execCount = 0;
    const inner = createExecRunner();
    const runner = {
      execute: async (input: Parameters<typeof inner.execute>[0]) => {
        execCount += 1;
        return inner.execute(input);
      },
    };
    const args = {
      runner,
      remoteCwd: rootDir,
      remoteDir,
      remotePath,
      body,
      label: "Process session remote script",
      action: "sync process session remote script",
      lockDir,
      timeoutMs: 30_000,
    } as const;

    // Cold start: the script is uploaded (single sha-gate exec that writes).
    const first = await syncRemoteTextFileWithHashSkip(args);
    expect(first.uploaded).toBe(true);
    await expect(readFile(remotePath, "utf8")).resolves.toBe(body);

    // Warm start: the remote hash matches, so the write is skipped entirely.
    execCount = 0;
    const second = await syncRemoteTextFileWithHashSkip(args);
    expect(second.uploaded).toBe(false);
    // A single hash-gate round-trip that performed 0 writes (down from ~3 execs).
    expect(execCount).toBe(1);
    // sha is still returned on the skip path so callers get a well-formed result.
    expect(second.sha256).toBe(first.sha256);
    // The remote file is unchanged and no upload/partial/lock leftovers remain.
    await expect(readFile(remotePath, "utf8")).resolves.toBe(body);
    await expect(
      readdir(remoteDir).then((entries) =>
        entries.filter(
          (entry) =>
            entry.endsWith(".paperclip-upload.b64") ||
            entry.endsWith(".partial") ||
            entry === ".paperclip-process-session-script.lock",
        ),
      ),
    ).resolves.toEqual([]);
  });

  it("test_process_session_script_rewritten_on_hash_mismatch: a mismatched remote hash still rewrites the script", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-hashskip-cold-"));
    cleanupDirs.push(rootDir);
    const remoteDir = path.join(rootDir, "runtime", "codex", "process-sessions");
    const remotePath = path.posix.join(remoteDir, "paperclip-process-session-remote.mjs");
    const lockDir = path.posix.join(remoteDir, ".paperclip-process-session-script.lock");
    const body = "console.log('process session remote script v2');\n";

    // Pre-seed the remote with a DIFFERENT script (a prior/stale build).
    await mkdir(remoteDir, { recursive: true });
    await writeFile(remotePath, "console.log('stale remote script');\n", "utf8");

    const result = await syncRemoteTextFileWithHashSkip({
      runner: createExecRunner(),
      remoteCwd: rootDir,
      remoteDir,
      remotePath,
      body,
      label: "Process session remote script",
      action: "sync process session remote script",
      lockDir,
      timeoutMs: 30_000,
    });

    expect(result.uploaded).toBe(true);
    await expect(readFile(remotePath, "utf8")).resolves.toBe(body);
  });

  it("fails loud when the hash-skip sync exec errors instead of silently re-uploading", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-hashskip-fail-"));
    cleanupDirs.push(rootDir);
    const remoteDir = path.join(rootDir, "runtime", "codex", "process-sessions");
    const remotePath = path.posix.join(remoteDir, "paperclip-process-session-remote.mjs");
    const lockDir = path.posix.join(remoteDir, ".paperclip-process-session-script.lock");

    // A runner whose exec fails: the hash-gate cannot be evaluated. The write
    // must surface the failure, never swallow it and re-upload behind a green
    // return value.
    const runner = {
      execute: async () => ({
        exitCode: 1,
        signal: null,
        timedOut: false,
        stdout: "",
        stderr: "hash gate boom",
        pid: null,
        startedAt: new Date().toISOString(),
      }),
    };

    await expect(
      syncRemoteTextFileWithHashSkip({
        runner,
        remoteCwd: rootDir,
        remoteDir,
        remotePath,
        body: "console.log('never written');\n",
        label: "Process session remote script",
        action: "sync process session remote script",
        lockDir,
        timeoutMs: 30_000,
      }),
    ).rejects.toThrow(/sync process session remote script/i);

    // Nothing was written to the remote path on the failure path.
    await expect(readFile(remotePath, "utf8")).rejects.toThrow();
  });

  it.each([
    "/api/companies/co-1/heartbeat-runs",
    "/api/companies/co-1/live-runs",
    "/api/heartbeat-runs/run-1",
    "/api/heartbeat-runs/run-1/events",
    "/api/heartbeat-runs/run-1/log",
    "/api/heartbeat-runs/run-1/workspace-operations",
    "/api/heartbeat-runs/run-1/issues",
    "/api/issues/issue-1/runs",
    "/api/issues/issue-1/recovery-actions",
  ])("admits only GET for diagnostic path %s on both transports", (requestPath) => {
    for (const routes of [undefined, HTTP2_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST]) {
      expect(authorizeSandboxCallbackBridgeRequestWithRoutes({ method: "GET", path: requestPath }, routes)).toBeNull();
      for (const method of ["POST", "PUT", "PATCH", "DELETE"]) {
        expect(authorizeSandboxCallbackBridgeRequestWithRoutes({ method, path: requestPath }, routes))
          .toBe(`Route not allowed: ${method} ${requestPath}`);
      }
      expect(authorizeSandboxCallbackBridgeRequestWithRoutes({ method: "GET", path: `${requestPath}/extra` }, routes))
        .toBe(`Route not allowed: GET ${requestPath}/extra`);
    }
  });

  it.each([
    { method: "POST", path: "/api/heartbeat-runs/run-1/cancel" },
    { method: "POST", path: "/api/heartbeat-runs/run-1/retry" },
    { method: "POST", path: "/api/heartbeat-runs/run-1/watchdog-decisions" },
    { method: "POST", path: "/api/issues/issue-1/recovery-actions/resolve" },
    { method: "POST", path: "/api/issues/issue-1/recovery-actions/retry-workspace-export" },
    { method: "GET", path: "/api/heartbeat-runs/run-1/provider-trace" },
    { method: "GET", path: "/api/companies/co-1/provider-traces" },
    { method: "GET", path: "/api/companies/co-1/execution-workspaces" },
  ])("keeps execution control and unrelated reads denied: $method $path", (request) => {
    expect(authorizeSandboxCallbackBridgeRequestWithRoutes(request))
      .toBe(`Route not allowed: ${request.method} ${request.path}`);
  });

  it("permits the documented heartbeat surface and denies unrelated routes", () => {
    const allowed: Array<{ method: string; path: string }> = [
      { method: "POST", path: "/api/companies/co-1/agent-commentary" },
      { method: "POST", path: "/runtime-tools/github/credentials" },
      { method: "GET", path: "/api/agents/me" },
      { method: "GET", path: "/api/agents/me/inbox-lite" },
      { method: "GET", path: "/api/agents/me/inbox/mine" },
      { method: "GET", path: "/api/agents/agent-1" },
      { method: "GET", path: "/api/agents/agent-1/skills" },
      { method: "POST", path: "/api/agents/agent-1/skills/sync" },
      { method: "PATCH", path: "/api/agents/agent-1/instructions-path" },
      { method: "GET", path: "/api/companies/co-1" },
      { method: "GET", path: "/api/companies/co-1/dashboard" },
      { method: "GET", path: "/api/companies/co-1/agents" },
      { method: "GET", path: "/api/companies/co-1/issues" },
      { method: "GET", path: "/api/companies/co-1/projects" },
      { method: "GET", path: "/api/companies/co-1/goals" },
      { method: "GET", path: "/api/companies/co-1/org" },
      { method: "GET", path: "/api/companies/co-1/approvals" },
      { method: "GET", path: "/api/companies/co-1/routines" },
      { method: "GET", path: "/api/companies/co-1/skills" },
      { method: "GET", path: "/api/companies/co-1/email/inboxes" },
      { method: "GET", path: "/api/companies/co-1/email/tasks/issue-1" },
      { method: "GET", path: "/api/companies/co-1/email/deliveries/send-1" },
      { method: "POST", path: "/api/companies/co-1/email/send" },
      // Hire skill (paperclip-create-agent): discovery + submit + issue linking
      { method: "GET", path: "/llms/agent-configuration.txt" },
      { method: "GET", path: "/llms/agent-configuration/claude_local.txt" },
      { method: "GET", path: "/llms/agent-icons.txt" },
      { method: "GET", path: "/api/companies/co-1/agent-configurations" },
      { method: "POST", path: "/api/companies/co-1/agent-hires" },
      { method: "POST", path: "/api/issues/issue-1/approvals" },
      { method: "GET", path: "/api/projects/proj-1" },
      { method: "GET", path: "/api/goals/goal-1" },
      { method: "GET", path: "/api/issues/issue-1" },
      { method: "GET", path: "/api/issues/issue-1/heartbeat-context" },
      { method: "GET", path: "/api/issues/issue-1/activity" },
      { method: "GET", path: "/api/issues/issue-1/comments" },
      { method: "GET", path: "/api/issues/issue-1/comments/c-1" },
      { method: "POST", path: "/api/issues/issue-1/comments" },
      { method: "GET", path: "/api/issues/issue-1/documents" },
      { method: "GET", path: "/api/issues/issue-1/documents/plan" },
      { method: "GET", path: "/api/issues/issue-1/documents/plan/revisions" },
      { method: "PUT", path: "/api/issues/issue-1/documents/plan" },
      { method: "POST", path: "/api/issues/issue-1/checkout" },
      { method: "POST", path: "/api/issues/issue-1/release" },
      { method: "PATCH", path: "/api/issues/issue-1" },
      { method: "GET", path: "/api/issues/issue-1/approvals" },
      { method: "GET", path: "/api/issues/issue-1/work-products" },
      { method: "POST", path: "/api/issues/issue-1/work-products" },
      { method: "PATCH", path: "/api/work-products/wp-1" },
      { method: "GET", path: "/api/issues/issue-1/interactions" },
      { method: "GET", path: "/api/issues/issue-1/interactions/inter-1" },
      { method: "POST", path: "/api/issues/issue-1/interactions" },
      { method: "POST", path: "/api/issues/issue-1/interactions/inter-1/accept" },
      { method: "POST", path: "/api/issues/issue-1/interactions/inter-1/reject" },
      { method: "POST", path: "/api/issues/issue-1/interactions/inter-1/respond" },
      { method: "POST", path: "/api/issues/issue-1/interactions/inter-1/verdicts" },
      { method: "POST", path: "/api/issues/issue-1/interactions/inter-1/withdraw" },
      { method: "POST", path: "/api/companies/co-1/issues" },
      { method: "GET", path: "/api/approvals/ap-1" },
      { method: "GET", path: "/api/approvals/ap-1/issues" },
      { method: "GET", path: "/api/approvals/ap-1/comments" },
      { method: "POST", path: "/api/approvals/ap-1/comments" },
      { method: "POST", path: "/api/companies/co-1/approvals" },
      { method: "GET", path: "/api/execution-workspaces/ws-1" },
      { method: "POST", path: "/api/execution-workspaces/ws-1/runtime-services/start" },
      { method: "POST", path: "/api/execution-workspaces/ws-1/runtime-services/stop" },
      { method: "POST", path: "/api/execution-workspaces/ws-1/runtime-services/restart" },
      { method: "GET", path: "/api/routines/r-1" },
      { method: "GET", path: "/api/routines/r-1/runs" },
      { method: "POST", path: "/api/companies/co-1/routines" },
      { method: "PATCH", path: "/api/routines/r-1" },
      { method: "POST", path: "/api/routines/r-1/run" },
      { method: "POST", path: "/api/routines/r-1/triggers" },
      { method: "PATCH", path: "/api/routine-triggers/t-1" },
      { method: "DELETE", path: "/api/routine-triggers/t-1" },
    ];
    for (const request of allowed) {
      expect(authorizeSandboxCallbackBridgeRequestWithRoutes(request)).toBeNull();
    }

    const denied: Array<{ method: string; path: string }> = [
      { method: "POST", path: "/api/issues/issue-1/activity" },
      { method: "GET", path: "/api/issues/issue-1/activity/extra" },
      { method: "GET", path: "/api/companies/co-1/activity" },
      { method: "GET", path: "/api/companies/co-1/audit/agent-actions" },
      { method: "GET", path: "/api/companies/co-1/agent-commentary" },
      { method: "POST", path: "/api/companies/co-1/agent-commentary/other" },
      { method: "POST", path: "/api/companies/co-1/email/inboxes" },
      { method: "POST", path: "/api/companies/co-1/email/connections" },
      { method: "POST", path: "/api/companies/co-1/email/inspect" },
      { method: "POST", path: "/api/companies/co-1/email/deliveries/send-1/resolve" },
      { method: "POST", path: "/api/email/inboxes/inbox-1/reconnect" },
      { method: "POST", path: "/api/email/inboxes/inbox-1/control" },
      { method: "DELETE", path: "/api/companies/co-1/email/tasks/issue-1" },
      { method: "DELETE", path: "/api/secrets" },
      // Pin the runtime-services regex to start/stop/restart only — anything
      // else (delete, reset, wipe, etc.) must stay denied even if the API
      // grows new actions later.
      { method: "POST", path: "/api/execution-workspaces/ws-1/runtime-services/delete" },
      { method: "POST", path: "/api/companies/co-1/agents" },
      // The hire allowlist must not over-match: only the exact .txt discovery
      // files, only agent-hires (not /agents), and no sub-resources beyond it.
      { method: "GET", path: "/llms/agent-configuration" },
      { method: "GET", path: "/llms/secrets.txt" },
      { method: "POST", path: "/api/companies/co-1/agent-hires/ap-1" },
      { method: "POST", path: "/api/issues/issue-1/approvals/ap-1" },
      { method: "POST", path: "/api/agents/agent-1/pause" },
      { method: "POST", path: "/api/agents/agent-1/terminate" },
      { method: "POST", path: "/api/agents/agent-1/keys" },
      { method: "POST", path: "/api/companies/co-1/exports" },
      { method: "POST", path: "/api/companies/co-1/imports/apply" },
      { method: "POST", path: "/api/companies/co-1/archive" },
      { method: "DELETE", path: "/api/issues/issue-1/documents/plan" },
      { method: "DELETE", path: "/api/issues/issue-1/approvals/ap-1" },
      { method: "DELETE", path: "/api/work-products/wp-1" },
      { method: "POST", path: "/api/approvals/ap-1/approve" },
      { method: "POST", path: "/api/approvals/ap-1/reject" },
      { method: "POST", path: "/api/companies/co-1/logo" },
      { method: "GET", path: "/api/companies/co-1/secrets" },
      { method: "PATCH", path: "/api/secrets/secret-1" },
    ];
    for (const request of denied) {
      expect(authorizeSandboxCallbackBridgeRequestWithRoutes(request)).toBe(
        `Route not allowed: ${request.method} ${request.path}`,
      );
    }

    // Both transports admit the full attachment workflow.
    const http2Allowed: Array<{ method: string; path: string }> = [
      { method: "GET", path: "/api/issues/issue-1/attachments" },
      { method: "POST", path: "/api/companies/co-1/issues/issue-1/attachments" },
      { method: "GET", path: "/api/attachments/att-1/content" },
    ];
    for (const request of http2Allowed) {
      expect(
        authorizeSandboxCallbackBridgeRequestWithRoutes(request, HTTP2_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST),
      ).toBeNull();
    }

    const http2Denied: Array<{ method: string; path: string }> = [
      // Wrong method for each attachment rule.
      { method: "GET", path: "/api/companies/co-1/issues/issue-1/attachments" },
      { method: "POST", path: "/api/attachments/att-1/content" },
      // Extra path segment for each attachment rule.
      { method: "POST", path: "/api/companies/co-1/issues/issue-1/attachments/att-1" },
      { method: "GET", path: "/api/attachments/att-1/content/extra" },
    ];
    for (const request of http2Denied) {
      expect(
        authorizeSandboxCallbackBridgeRequestWithRoutes(request, HTTP2_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST),
      ).toBe(`Route not allowed: ${request.method} ${request.path}`);
    }
  });

  it("admits listing, uploads and downloads on the default queue route list", () => {
    const attachmentRequests: Array<{ method: string; path: string }> = [
      { method: "GET", path: "/api/issues/issue-1/attachments" },
      { method: "POST", path: "/api/companies/co-1/issues/issue-1/attachments" },
      { method: "GET", path: "/api/attachments/att-1/content" },
    ];
    for (const request of attachmentRequests) {
      expect(authorizeSandboxCallbackBridgeRequestWithRoutes(request)).toBeNull();
    }
  });

  it.each([
    [4 * 60 * 60 * 1000, 30_000],
    [250, 250],
  ])("bounds a hung bridge read configured for %i ms to %i ms", async (configuredMs, expectedMs) => {
    vi.useFakeTimers();
    try {
      const runner = { execute: vi.fn(() => new Promise<RunProcessResult>(() => {})) };
      const client = createCommandManagedSandboxCallbackBridgeQueueClient({
        runner, remoteCwd: "/workspace", timeoutMs: configuredMs,
      });
      let error: unknown;
      const read = client.readTextFile("/workspace/events/1.json").catch((caught) => { error = caught; });
      await vi.advanceTimersByTimeAsync(expectedMs - 1);
      expect(error).toBeUndefined();
      await vi.advanceTimersByTimeAsync(1);
      expect(error).toEqual(new Error(`Sandbox bridge control command timed out after ${expectedMs}ms.`));
      await read;
      expect(runner.execute).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
        timeoutMs: expectedMs, bypassSession: true,
      }));
      expect(vi.getTimerCount()).toBe(0);
    } finally {
      vi.useRealTimers();
    }
  });

  it("abandons a timed-out upload without replaying it or continuing after a late response", async () => {
    vi.useFakeTimers();
    try {
      const success: RunProcessResult = {
        exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null,
        startedAt: new Date().toISOString(),
      };
      let finishAppend!: (result: RunProcessResult) => void;
      const runner = {
        execute: vi.fn(async (input: { args?: string[] }) => {
          if (input.args?.[1]?.startsWith("printf")) {
            return new Promise<RunProcessResult>((resolve) => { finishAppend = resolve; });
          }
          return success;
        }),
      };
      const client = createCommandManagedSandboxCallbackBridgeQueueClient({
        runner, remoteCwd: "/workspace", timeoutMs: 4 * 60 * 60 * 1000,
      });
      let error: unknown;
      const write = client.writeTextFile("/workspace/stdin/1.json", "sensitive-input")
        .catch((caught) => { error = caught; });
      await vi.advanceTimersByTimeAsync(30_000);
      expect(error).toEqual(new Error("Sandbox bridge control command timed out after 30000ms."));
      await write;
      finishAppend(success);
      await vi.advanceTimersByTimeAsync(0);
      const scripts = runner.execute.mock.calls.map(([input]) => input.args?.[1] ?? "");
      expect(scripts.filter((script) => script.startsWith("printf"))).toHaveLength(1);
      expect(scripts.some((script) => script.startsWith("base64 -d"))).toBe(false);
      expect(scripts.at(-1)).toMatch(/^rm -f .*paperclip-upload/);
      expect(vi.getTimerCount()).toBe(0);
    } finally {
      vi.useRealTimers();
    }
  });

  it.each(["start", "stop"])("bounds a hung callback bridge %s while preserving its launch environment", async (stage) => {
    vi.useFakeTimers();
    try {
      const runner = {
        execute: vi.fn(async (input: { args?: string[]; env?: Record<string, string> }) => {
          const script = input.args?.[1] ?? "";
          if ((stage === "start" && script.includes("nohup")) ||
              (stage === "stop" && script.includes('kill "$pid"'))) {
            return new Promise<RunProcessResult>(() => {});
          }
          return {
            exitCode: 0, signal: null, timedOut: false, stderr: "", pid: null,
            startedAt: new Date().toISOString(), stdout: JSON.stringify({ port: 3101 }),
          };
        }),
      };
      let error: unknown;
      const operation = startSandboxCallbackBridgeServer({
        runner, remoteCwd: "/workspace", assetRemoteDir: "/workspace/assets",
        queueDir: "/workspace/queue", bridgeToken: "private-bridge-token", timeoutMs: 4 * 60 * 60 * 1000,
      }).then(async (bridge) => { if (stage === "stop") await bridge.stop(); })
        .catch((caught) => { error = caught; });
      await vi.advanceTimersByTimeAsync(30_000);
      expect(error).toEqual(new Error("Sandbox bridge control command timed out after 30000ms."));
      await operation;
      expect(runner.execute.mock.calls[0]?.[0].env).toMatchObject({
        PAPERCLIP_BRIDGE_TOKEN: "private-bridge-token",
        PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge",
      });
      expect(runner.execute).toHaveBeenLastCalledWith(expect.objectContaining({ timeoutMs: 30_000 }));
      expect(vi.getTimerCount()).toBe(0);
    } finally {
      vi.useRealTimers();
    }
  });

  it("marks command-managed bridge operations with the bridge execution channel", async () => {
    const runner = {
      execute: vi.fn(async () => ({
        exitCode: 0,
        signal: null,
        timedOut: false,
        stdout: "",
        stderr: "",
        pid: null,
        startedAt: new Date().toISOString(),
      })),
    };

    const client = createCommandManagedSandboxCallbackBridgeQueueClient({
      runner,
      remoteCwd: "/workspace",
      timeoutMs: 30_000,
    });

    await client.makeDir("/workspace/.paperclip-runtime/codex/paperclip-bridge/queue");

    expect(runner.execute).toHaveBeenCalledWith(expect.objectContaining({
      env: {
        PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge",
      },
    }));
  });

  it("creates the bridge queue directories in one directory-creation exec", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-makedirs-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const makeDir = vi.fn(async () => {});
    const makeDirs = vi.fn(async () => {});

    const worker = await startSandboxCallbackBridgeWorker({
      client: {
        makeDir,
        makeDirs,
        listJsonFiles: async () => [],
        readTextFile: async () => {
          throw new Error("unexpected readTextFile");
        },
        writeTextFile: async () => {},
        rename: async () => {},
        remove: async () => {},
      },
      queueDir,
      authorizeRequest: async () => null,
      handleRequest: async () => ({ status: 200, body: "ok" }),
    });

    await worker.stop();

    expect(makeDir).not.toHaveBeenCalled();
    expect(makeDirs).toHaveBeenCalledTimes(1);
    expect(makeDirs).toHaveBeenCalledWith([
      directories.rootDir,
      directories.requestsDir,
      directories.responsesDir,
      directories.logsDir,
    ]);
  });

  it("falls back to sequential makeDir when the queue client omits makeDirs", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-makedir-fallback-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const makeDir = vi.fn(async (_remotePath: string) => {});

    // A queue client that predates the batched makeDirs method. The worker
    // must still create every queue directory through sequential makeDir.
    const worker = await startSandboxCallbackBridgeWorker({
      client: {
        makeDir,
        listJsonFiles: async () => [],
        readTextFile: async () => {
          throw new Error("unexpected readTextFile");
        },
        writeTextFile: async () => {},
        rename: async () => {},
        remove: async () => {},
      },
      queueDir,
      authorizeRequest: async () => null,
      handleRequest: async () => ({ status: 200, body: "ok" }),
    });

    await worker.stop();

    expect(makeDir.mock.calls.map((call) => call[0])).toEqual([
      directories.rootDir,
      directories.requestsDir,
      directories.responsesDir,
      directories.logsDir,
    ]);
  });

  it("runs one mkdir -p exec for makeDirs on the command-managed queue client", async () => {
    const runner = {
      execute: vi.fn(async (_input: { args?: string[] }) => ({
        exitCode: 0,
        signal: null,
        timedOut: false,
        stdout: "",
        stderr: "",
        pid: null,
        startedAt: new Date().toISOString(),
      })),
    };

    const client = createCommandManagedSandboxCallbackBridgeQueueClient({
      runner,
      remoteCwd: "/workspace",
      timeoutMs: 30_000,
    });

    // The command-managed client always provides the batched makeDirs method.
    expect(client.makeDirs).toBeDefined();
    await client.makeDirs?.(["/workspace/a", "/workspace/b", "/workspace/c"]);

    expect(runner.execute).toHaveBeenCalledTimes(1);
    const call = runner.execute.mock.calls[0][0];
    const script = call.args?.[call.args.length - 1] ?? "";
    expect(script).toContain("mkdir -p");
    expect(script).toContain("/workspace/a");
    expect(script).toContain("/workspace/b");
    expect(script).toContain("/workspace/c");
  });

  // Capture the run-level error the worker surfaces through `runtimeSpan`. The
  // worker runs a throwing function under the `sandbox.callbackBridge.workerFailed`
  // span; this double records that error and re-throws, like the real runner.
  function createWorkerErrorCapture(): {
    runtimeSpan: RuntimeSpanRunner;
    workerErrors: string[];
  } {
    const workerErrors: string[] = [];
    const runtimeSpan: RuntimeSpanRunner = async (name, work) => {
      try {
        return await work();
      } catch (error) {
        if (name === "sandbox.callbackBridge.workerFailed") {
          workerErrors.push(error instanceof Error ? error.message : String(error));
        }
        throw error;
      }
    };
    return { runtimeSpan, workerErrors };
  }

  function bridgeRequestJson(id: string): string {
    return `${JSON.stringify({
      id,
      method: "GET",
      path: "/api/agents/me",
      query: "",
      headers: {},
      body: "",
      createdAt: new Date().toISOString(),
    })}\n`;
  }

  it("times out a stalled poll, surfaces a run-level error, and recovers to deliver the request", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-hang-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    await mkdir(directories.requestsDir, { recursive: true });
    await publishBridgeRequest(path.posix.join(directories.requestsDir, "req-a.json"), bridgeRequestJson("req-a"));

    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();

    const base = createFileSystemSandboxCallbackBridgeQueueClient();
    let listCalls = 0;
    const client: SandboxCallbackBridgeQueueClient = {
      ...base,
      // The first poll never resolves — a silently unresponsive sandbox channel.
      // The per-iteration timeout must convert the hang into a caught error, and
      // the loop must then back off and retry rather than die: the request never
      // reached the handler, so the retry delivers the real response. A
      // sustained outage is the watchdog's job (proven separately below), not a
      // reason to fail a request one transient hang could still serve.
      listJsonFiles: async (dir) => {
        listCalls += 1;
        if (listCalls === 1) {
          return await new Promise<string[]>(() => {});
        }
        return await base.listJsonFiles(dir);
      },
    };

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 50,
      watchdogTimeoutMs: 10_000,
      runtimeSpan,
      authorizeRequest: async () => null,
      handleRequest: async () => ({ status: 200, body: "ok" }),
    });

    const responseFile = await waitForJsonFile(directories.responsesDir, 3_000);
    const responseBody = await readFile(path.posix.join(directories.responsesDir, responseFile), "utf8");
    expect(JSON.parse(responseBody).status).toBe(200);
    expect(workerErrors.length).toBeGreaterThan(0);
    expect(workerErrors[0]).toContain("timed out");

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("does not abandon an in-flight handler, so a started mutation is not applied twice", async () => {
    // Prove the completion fence for a request whose handler already started. The
    // per-iteration timeout rejects the wrapper but does not stop the handler, so
    // the host operation (a mutation) is still in flight. The recovery path must
    // not write a 503 there; a 503 makes the caller retry while the original
    // mutation still completes, applying it twice. The test proves no 503 lands
    // and the handler's real response is delivered exactly once.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-late.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    // An in-memory queue client with no file-existence guards. A response write
    // always lands here, so only the completion fence controls the outcome. The
    // real filesystem and command clients add their own existence guards; this
    // client removes them, so the test isolates the fence.
    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-late"));
    const responseWrites: Array<{ path: string; status: number }> = [];
    const requestRemovals: string[] = [];

    const handlerControl: { release: (() => void) | null } = { release: null };
    let handlerCompleted = false;

    const client: SandboxCallbackBridgeQueueClient = {
      ...virtualQueueReadClient(directories, requestBodies),
      writeResponseFile: async (remotePath, body) => {
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push(remotePath);
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 50,
      watchdogTimeoutMs: 10_000,
      runtimeSpan,
      authorizeRequest: async () => null,
      // The handler stays pending until the test releases it, well after the
      // per-iteration timeout fires. It records that it finished, so the test
      // proves the in-flight handler truly ran.
      handleRequest: () =>
        new Promise<{ status: number; body?: string }>((resolve) => {
          handlerControl.release = () => {
            handlerCompleted = true;
            resolve({ status: 200, body: "req-late" });
          };
        }),
    });

    // Wait until the per-iteration timeout surfaced a run error and the handler
    // is pending. The recovery path ran, so it already skipped the in-flight
    // request.
    await waitFor(
      () => workerErrors.some((message) => message.includes("timed out")) && handlerControl.release !== null,
      3_000,
    );
    // The recovery path wrote no 503 for the in-flight request.
    expect(responseWrites.some((write) => write.status === 503)).toBe(false);

    // Release the handler after the timeout. It delivers the real response.
    handlerControl.release?.();
    await waitFor(() => handlerCompleted, 3_000);
    // Give the finalize write and remove time to land.
    await new Promise((resolve) => setTimeout(resolve, 50));

    expect(handlerCompleted).toBe(true);
    // The handler committed exactly one response for the request: the real 200.
    expect(responseWrites.filter((write) => write.path === responsePath)).toEqual([
      { path: responsePath, status: 200 },
    ]);
    expect(responseWrites.some((write) => write.status === 503)).toBe(false);
    // The handler removed the request file exactly once, after it finalized.
    expect(requestRemovals).toEqual([requestPath]);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("aborts an in-flight handler on timeout, so a cooperating handler finalizes instead of stranding the request", async () => {
    // A handler that threads the worker signal into its work must stop when the
    // per-iteration timeout fires. It then finalizes with its own error
    // response, so the request does not strand with no response. The recovery
    // path still writes no 503 for the handler-owned request, so a started
    // mutation is not applied twice. The abort reaches the handler after the host
    // operation started, so the handler finalizes a non-retryable 504, not a
    // retryable 502; the caller then does not retry a mutation that may have
    // committed.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-abort.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-abort"));
    const responseWrites: Array<{ path: string; status: number }> = [];
    const requestRemovals: string[] = [];

    const client: SandboxCallbackBridgeQueueClient = {
      ...virtualQueueReadClient(directories, requestBodies),
      writeResponseFile: async (remotePath, body) => {
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push(remotePath);
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();
    let handlerAborted = false;

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 50,
      watchdogTimeoutMs: 10_000,
      runtimeSpan,
      authorizeRequest: async () => null,
      // The handler stays pending until the worker aborts the signal. It then
      // rejects, so the request finalizes with the handler's own error response.
      handleRequest: (_request, options) =>
        new Promise<{ status: number; body?: string }>((_resolve, reject) => {
          options?.signal.addEventListener("abort", () => {
            handlerAborted = true;
            reject(new Error("aborted by worker"));
          });
        }),
    });

    // The handler finalizes only after the worker aborts it. The request gets a
    // terminal response (a 502 from the handler failure), never stranded.
    await waitFor(() => responseWrites.some((write) => write.path === responsePath), 3_000);

    expect(handlerAborted).toBe(true);
    expect(workerErrors.some((message) => message.includes("timed out"))).toBe(true);
    // Exactly one response landed: the handler's non-retryable 504. The recovery
    // path wrote no competing 503, and the aborted handler wrote no retryable 502.
    expect(responseWrites.filter((write) => write.path === responsePath)).toEqual([
      { path: responsePath, status: 504 },
    ]);
    expect(responseWrites.some((write) => write.status === 503)).toBe(false);
    expect(responseWrites.some((write) => write.status === 502)).toBe(false);
    // The handler removed the request file after it finalized. The recovery path
    // may issue a redundant idempotent remove for the same path when the handler
    // finalized first, so assert the removal happened rather than a fixed count.
    expect(requestRemovals).toContain(requestPath);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("finalizes an aborted handler that ignores the signal and never settles with a non-retryable 504 backstop", async () => {
    // A handler that does not thread the worker signal into its work and never
    // settles must not strand the request. The recovery path aborts the handler,
    // waits the grace, then writes a non-retryable 504 backstop, so the request
    // gets a terminal response even when the handler ignores the abort. The 504
    // is non-retryable, so the caller does not retry a mutation that may have
    // committed.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-stuck.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-stuck"));
    const responseWrites: Array<{ path: string; status: number; body: string }> = [];
    const requestRemovals: string[] = [];

    const client: SandboxCallbackBridgeQueueClient = {
      ...virtualQueueReadClient(directories, requestBodies),
      writeResponseFile: async (remotePath, body) => {
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status, body });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push(remotePath);
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();
    let handlerStarted = false;

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 50,
      watchdogTimeoutMs: 10_000,
      // A short grace, so the backstop fires quickly after the abort.
      abortedHandlerGraceMs: 30,
      runtimeSpan,
      authorizeRequest: async () => null,
      // The handler ignores the worker signal and never settles. Without the
      // backstop, the request would stay without a response forever.
      handleRequest: () => {
        handlerStarted = true;
        return new Promise<{ status: number; body?: string }>(() => {});
      },
    });

    // The backstop writes the terminal response after the grace.
    await waitFor(() => responseWrites.some((write) => write.path === responsePath), 3_000);

    expect(handlerStarted).toBe(true);
    expect(workerErrors.some((message) => message.includes("timed out"))).toBe(true);
    // Exactly one response landed: the non-retryable 504 backstop. The recovery
    // path wrote no retryable 503 or 502 for the handler-owned request.
    const requestResponses = responseWrites.filter((write) => write.path === responsePath);
    expect(requestResponses).toHaveLength(1);
    expect(requestResponses[0]?.status).toBe(504);
    const parsed = JSON.parse(requestResponses[0]!.body.trim());
    const responseBody = JSON.parse(parsed.body);
    expect(responseBody.outcome).toBe("indeterminate");
    expect(responseBody.retryable).toBe(false);
    expect(parsed.headers?.["x-paperclip-bridge-outcome"]).toBe("indeterminate");
    expect(responseWrites.some((write) => write.status === 503)).toBe(false);
    expect(responseWrites.some((write) => write.status === 502)).toBe(false);
    // The backstop removed the request file, so it does not strand in the queue.
    expect(requestRemovals).toContain(requestPath);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("finalizes a late worker-aborted handler with a non-retryable 504, so the caller does not retry a committed mutation", async () => {
    // Prove the completion status for a mutating request that the worker aborts.
    // The per-iteration timeout aborts a handler that already started its host
    // operation, so the mutation may have committed. The bridge cannot cancel a
    // host operation that is in flight. The handler completes late, after the
    // abort. Its response must be a non-retryable 504, never a retryable 502 or
    // 503; a retry of either would apply the mutation twice.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-late-abort.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-late-abort"));
    const responseWrites: Array<{ path: string; status: number; body: string }> = [];
    const requestRemovals: string[] = [];

    const client: SandboxCallbackBridgeQueueClient = {
      ...virtualQueueReadClient(directories, requestBodies),
      writeResponseFile: async (remotePath, body) => {
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status, body });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push(remotePath);
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();
    let handlerCompletedLate = false;

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 50,
      watchdogTimeoutMs: 10_000,
      runtimeSpan,
      authorizeRequest: async () => null,
      // The handler mirrors a mutating forward that already committed on the
      // host. It rejects only after the worker aborts it, so its completion is
      // late. The mutation stays committed; the abort cannot undo it.
      handleRequest: (_request, options) =>
        new Promise<{ status: number; body?: string }>((_resolve, reject) => {
          options?.signal.addEventListener("abort", () => {
            handlerCompletedLate = true;
            reject(new Error("aborted by worker after the mutation committed"));
          });
        }),
    });

    await waitFor(() => responseWrites.some((write) => write.path === responsePath), 3_000);

    expect(handlerCompletedLate).toBe(true);
    expect(workerErrors.some((message) => message.includes("timed out"))).toBe(true);

    // Exactly one response landed for the request: the non-retryable 504.
    const requestResponses = responseWrites.filter((write) => write.path === responsePath);
    expect(requestResponses).toHaveLength(1);
    expect(requestResponses[0]?.status).toBe(504);

    // The bridge wrote no retryable status for the possibly-committed mutation.
    expect(responseWrites.some((write) => write.status === 502)).toBe(false);
    expect(responseWrites.some((write) => write.status === 503)).toBe(false);

    // The response marks the outcome indeterminate and non-retryable, so the
    // caller does not retry.
    const parsed = JSON.parse(requestResponses[0]?.body ?? "{}");
    expect(parsed.status).toBe(504);
    const responseBody = JSON.parse(parsed.body);
    expect(responseBody.outcome).toBe("indeterminate");
    expect(responseBody.retryable).toBe(false);
    expect(parsed.headers?.["x-paperclip-bridge-outcome"]).toBe("indeterminate");

    expect(requestRemovals).toContain(requestPath);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("retries a failed 504 backstop write and keeps the request until it lands, so the caller is not stranded", async () => {
    // A backstop write can fail transiently, or it can exceed the iteration
    // timeout. The recovery path must not drop the request file on that failure. A
    // dropped file leaves no terminal 504 for the caller and fences out a late
    // handler, so the caller waits to its own deadline and gets a generic 502. The
    // recovery path retries the write and removes the request file only after the
    // write lands.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-retry.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-retry"));
    const responseWrites: Array<{ path: string; status: number; body: string }> = [];
    const requestRemovals: Array<{ path: string; afterWrites: number }> = [];
    let writeAttempts = 0;

    const client: SandboxCallbackBridgeQueueClient = {
      ...virtualQueueReadClient(directories, requestBodies),
      writeResponseFile: async (remotePath, body) => {
        writeAttempts += 1;
        // The first backstop write fails; the retry then succeeds.
        if (writeAttempts === 1) {
          throw new Error("simulated transient write failure");
        }
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status, body });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push({ path: remotePath, afterWrites: writeAttempts });
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan } = createWorkerErrorCapture();

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 50,
      watchdogTimeoutMs: 10_000,
      // A short grace, so the backstop fires quickly after the abort.
      abortedHandlerGraceMs: 30,
      runtimeSpan,
      authorizeRequest: async () => null,
      // The handler ignores the worker signal and never settles, so only the
      // backstop can finalize the request.
      handleRequest: () => new Promise<{ status: number; body?: string }>(() => {}),
    });

    // The backstop retries the write, so the terminal 504 lands after the failure.
    await waitFor(() => responseWrites.some((write) => write.path === responsePath), 3_000);

    const requestResponses = responseWrites.filter((write) => write.path === responsePath);
    expect(requestResponses).toHaveLength(1);
    expect(requestResponses[0]?.status).toBe(504);
    // The write failed once, so the backstop wrote on a later attempt.
    expect(writeAttempts).toBeGreaterThanOrEqual(2);
    // The recovery path removed the request file only after the write landed. The
    // failed first attempt kept the file, so the request never dropped with no
    // response.
    const requestPathRemovals = requestRemovals.filter((entry) => entry.path === requestPath);
    expect(requestPathRemovals.length).toBeGreaterThanOrEqual(1);
    for (const entry of requestPathRemovals) {
      expect(entry.afterWrites).toBeGreaterThanOrEqual(2);
    }
    // The bridge wrote no retryable status for the possibly-committed mutation.
    expect(responseWrites.some((write) => write.status === 503)).toBe(false);
    expect(responseWrites.some((write) => write.status === 502)).toBe(false);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("retries a handler response write that fails transiently and delivers the real response, not a retryable 503", async () => {
    // A handler settles with its own response, but the terminal write fails
    // once. `finalize` must retry the write and deliver the real response. It
    // must not fence out recovery and leave the request for a retryable 503,
    // which would let the caller repeat a possibly-committed mutation.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-write-retry.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-write-retry"));
    const responseWrites: Array<{ path: string; status: number; body: string }> = [];
    const requestRemovals: string[] = [];
    let writeAttempts = 0;

    const client: SandboxCallbackBridgeQueueClient = {
      ...virtualQueueReadClient(directories, requestBodies),
      writeResponseFile: async (remotePath, body) => {
        writeAttempts += 1;
        // The first handler-response write fails; the retry then succeeds.
        if (writeAttempts === 1) {
          throw new Error("simulated transient response write failure");
        }
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status, body });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push(remotePath);
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan } = createWorkerErrorCapture();

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 500,
      watchdogTimeoutMs: 10_000,
      runtimeSpan,
      authorizeRequest: async () => null,
      // The handler settles with its own 200 response.
      handleRequest: async () => ({ status: 200, body: JSON.stringify({ ok: true }) }),
    });

    await waitFor(() => responseWrites.some((write) => write.path === responsePath), 3_000);

    // Exactly one response landed: the handler's real 200. The retry delivered it
    // after the transient failure.
    const requestResponses = responseWrites.filter((write) => write.path === responsePath);
    expect(requestResponses).toHaveLength(1);
    expect(requestResponses[0]?.status).toBe(200);
    expect(writeAttempts).toBeGreaterThanOrEqual(2);
    // The bridge wrote no retryable status, so the caller does not repeat the
    // request.
    expect(responseWrites.some((write) => write.status === 503)).toBe(false);
    expect(responseWrites.some((write) => write.status === 502)).toBe(false);
    // The handler removed the request file after the write landed.
    expect(requestRemovals).toContain(requestPath);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("re-arms the 504 backstop when every handler response write fails, so the caller gets a terminal 504 instead of stranding", async () => {
    // A handler settles with its own response, but every terminal write fails.
    // `finalize` must roll back the fence and re-arm the 504 backstop, so the
    // recovery still delivers a non-retryable terminal response. Without the
    // re-arm, a rejected write leaves the request for a retryable 503 and a hung
    // write strands the caller until its own deadline.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-write-backstop.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-write-backstop"));
    const responseWrites: Array<{ path: string; status: number; body: string }> = [];
    const requestRemovals: string[] = [];
    let writeAttempts = 0;

    const client: SandboxCallbackBridgeQueueClient = {
      ...virtualQueueReadClient(directories, requestBodies),
      writeResponseFile: async (remotePath, body) => {
        writeAttempts += 1;
        // Fail every `finalize` write attempt. The re-armed 504 backstop then
        // writes on a later attempt.
        if (writeAttempts <= 3) {
          throw new Error("simulated persistent response write failure");
        }
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status, body });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push(remotePath);
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan } = createWorkerErrorCapture();

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 500,
      watchdogTimeoutMs: 10_000,
      // A short grace, so the re-armed backstop fires quickly.
      abortedHandlerGraceMs: 30,
      runtimeSpan,
      authorizeRequest: async () => null,
      // The handler settles with its own 200 response, a committed mutation.
      handleRequest: async () => ({ status: 200, body: JSON.stringify({ ok: true }) }),
    });

    // The re-armed backstop writes the terminal 504 after the finalize writes fail.
    await waitFor(() => responseWrites.some((write) => write.path === responsePath), 3_000);

    const requestResponses = responseWrites.filter((write) => write.path === responsePath);
    expect(requestResponses).toHaveLength(1);
    // The backstop delivered a non-retryable 504 with an indeterminate outcome.
    expect(requestResponses[0]?.status).toBe(504);
    const parsed = JSON.parse(requestResponses[0]?.body ?? "{}");
    const responseBody = JSON.parse(parsed.body);
    expect(responseBody.outcome).toBe("indeterminate");
    expect(responseBody.retryable).toBe(false);
    expect(parsed.headers?.["x-paperclip-bridge-outcome"]).toBe("indeterminate");
    // The finalize writes failed before the backstop wrote, so it took a later
    // attempt.
    expect(writeAttempts).toBeGreaterThanOrEqual(4);
    // The bridge wrote no retryable status for the possibly-committed mutation.
    expect(responseWrites.some((write) => write.status === 503)).toBe(false);
    expect(responseWrites.some((write) => write.status === 502)).toBe(false);
    // The backstop removed the request file, so it does not strand in the queue.
    expect(requestRemovals).toContain(requestPath);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("retries a recovery 503 write that fails transiently, delivers the 503, and removes the request", async () => {
    // A request attempt times out (its first read hangs), so the loop's request
    // catch runs the recovery pass, which aborts the queued request with a 503.
    // The first 503 write fails, so the recovery must retry it inside the same
    // pass. It then delivers the 503 and removes the request. The request is
    // unclaimed, so its host mutation never ran; the 503 stays retry-safe. (A
    // hung poll no longer triggers this pass — the loop backs off and retries
    // the poll instead, and a sustained hang is the watchdog's job.)
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-503-retry.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-503-retry"));
    const responseWrites: Array<{ path: string; status: number; body: string }> = [];
    const requestRemovals: string[] = [];
    let readCalls = 0;
    let writeAttempts = 0;

    const client: SandboxCallbackBridgeQueueClient = {
      makeDir: async () => {},
      makeDirs: async () => {},
      listJsonFiles: async (dir) =>
        dir === directories.requestsDir
          ? [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort()
          : [],
      // The first read never resolves — a silently unresponsive sandbox channel
      // hit mid-request, before the handler claim. The per-iteration timeout
      // converts the hang into a caught error, so the loop's request catch runs
      // the recovery pass. The recovery's own read resolves, so it can build and
      // deliver the 503.
      readTextFile: async (remotePath) => {
        readCalls += 1;
        if (readCalls === 1) {
          return await new Promise<string>(() => {});
        }
        const body = requestBodies.get(remotePath);
        if (body === undefined) {
          throw new Error(`missing request ${remotePath}`);
        }
        return body;
      },
      writeTextFile: async () => {},
      writeResponseFile: async (remotePath, body) => {
        writeAttempts += 1;
        // The first recovery 503 write fails; the retry then succeeds.
        if (writeAttempts === 1) {
          throw new Error("simulated transient recovery 503 write failure");
        }
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status, body });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push(remotePath);
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan } = createWorkerErrorCapture();

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 200,
      watchdogTimeoutMs: 10_000,
      runtimeSpan,
      authorizeRequest: async () => null,
      handleRequest: async () => ({ status: 200, body: "ok" }),
    });

    await waitFor(() => responseWrites.some((write) => write.path === responsePath), 3_000);

    // Exactly one response landed: the retry-safe 503. The retry delivered it
    // after the transient failure.
    const requestResponses = responseWrites.filter((write) => write.path === responsePath);
    expect(requestResponses).toHaveLength(1);
    expect(requestResponses[0]?.status).toBe(503);
    expect(writeAttempts).toBeGreaterThanOrEqual(2);
    // The recovery removed the request file only after the 503 write landed.
    expect(requestRemovals).toContain(requestPath);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("keeps the queued request when every recovery 503 write fails, so a later pass can still deliver a terminal 503", async () => {
    // The poll times out, so the recovery path aborts the queued request with a
    // 503. Every 503 write fails. The recovery must keep the request file instead
    // of dropping it. Without the fix, the recovery removed the request before its
    // single write attempt, so a failed write left no queue state; a later
    // recovery pass found nothing and the caller waited until its own deadline.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-503-keep.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-503-keep"));
    const responseWrites: Array<{ path: string; status: number; body: string }> = [];
    const requestRemovals: string[] = [];
    let listCalls = 0;
    let writeAttempts = 0;

    const client: SandboxCallbackBridgeQueueClient = {
      makeDir: async () => {},
      makeDirs: async () => {},
      listJsonFiles: async (dir) => {
        if (dir !== directories.requestsDir) {
          return [];
        }
        listCalls += 1;
        if (listCalls === 1) {
          return await new Promise<string[]>(() => {});
        }
        return [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort();
      },
      readTextFile: async (remotePath) => {
        const body = requestBodies.get(remotePath);
        if (body === undefined) {
          throw new Error(`missing request ${remotePath}`);
        }
        return body;
      },
      writeTextFile: async () => {},
      writeResponseFile: async () => {
        writeAttempts += 1;
        // Fail every recovery 503 write attempt.
        throw new Error("simulated persistent recovery 503 write failure");
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestRemovals.push(remotePath);
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan } = createWorkerErrorCapture();

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 200,
      watchdogTimeoutMs: 10_000,
      runtimeSpan,
      authorizeRequest: async () => null,
      handleRequest: async () => ({ status: 200, body: "ok" }),
    });

    // Wait until the recovery pass has exhausted its bounded 503 write attempts
    // (three, the same bound as the finalize and 504 backstop writes).
    await waitFor(() => writeAttempts >= 3, 3_000);

    // The recovery could not write the 503, so it kept the request file. The
    // request still sits in the queue for a later recovery pass to deliver a
    // terminal 503. A drop here would strand the caller until its own deadline.
    expect(requestRemovals).not.toContain(requestPath);
    expect(requestBodies.has(requestPath)).toBe(true);
    // No response landed, because every write failed. The recovery wrote no
    // partial or retryable status for the possibly-committed mutation.
    expect(responseWrites).toHaveLength(0);

    await worker.stop({ drainTimeoutMs: 10 });
  });


  it("abandons a request before its handler starts, so the mutation never runs", async () => {
    // Prove the reverse race. When the recovery path claims a request before the
    // handler starts its host operation, the handler must bail without running
    // the mutation. A retry after the 503 then applies the mutation once.
    const waitFor = createWorkerWaiter();

    const queueDir = "/virtual-bridge/queue";
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const requestFile = "req-early.json";
    const requestPath = path.posix.join(directories.requestsDir, requestFile);
    const responsePath = path.posix.join(directories.responsesDir, requestFile);

    const requestBodies = new Map<string, string>();
    requestBodies.set(requestPath, bridgeRequestJson("req-early"));
    const responseWrites: Array<{ path: string; status: number }> = [];

    const client: SandboxCallbackBridgeQueueClient = {
      ...virtualQueueReadClient(directories, requestBodies),
      writeResponseFile: async (remotePath, body) => {
        responseWrites.push({ path: remotePath, status: JSON.parse(body.trim()).status });
        return { wrote: true };
      },
      rename: async () => {},
      remove: async (remotePath) => {
        requestBodies.delete(remotePath);
      },
    };

    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();

    // The authorize step stays pending until the test releases it, so the handler
    // does not start before the per-iteration timeout fires and the recovery path
    // claims the request.
    const authorizeControl: { release: (() => void) | null } = { release: null };
    let handlerCalls = 0;

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      iterationTimeoutMs: 50,
      watchdogTimeoutMs: 10_000,
      runtimeSpan,
      authorizeRequest: () =>
        new Promise<string | null>((resolve) => {
          authorizeControl.release = () => resolve(null);
        }),
      handleRequest: async () => {
        handlerCalls += 1;
        return { status: 200, body: "req-early" };
      },
    });

    // Wait until the recovery path wrote the 503 and the authorize step is
    // pending.
    await waitFor(
      () => responseWrites.some((write) => write.status === 503) && authorizeControl.release !== null,
      3_000,
    );
    expect(responseWrites.some((write) => write.path === responsePath && write.status === 503)).toBe(true);

    // Release authorize after the 503. The handler must bail at its claim.
    authorizeControl.release?.();
    await new Promise((resolve) => setTimeout(resolve, 50));

    // The handler never ran, so the mutation never applied.
    expect(handlerCalls).toBe(0);
    // No competing 200 landed over the 503.
    expect(responseWrites.some((write) => write.path === responsePath && write.status === 200)).toBe(false);
    expect(workerErrors.some((message) => message.includes("timed out"))).toBe(true);

    await worker.stop({ drainTimeoutMs: 10 });
  });

  it("trips the watchdog on a stalled poll, writes a 503, and surfaces a run-level error", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-watchdog-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    await mkdir(directories.requestsDir, { recursive: true });
    await publishBridgeRequest(path.posix.join(directories.requestsDir, "req-w.json"), bridgeRequestJson("req-w"));

    const base = createFileSystemSandboxCallbackBridgeQueueClient();
    let listCalls = 0;
    const client: SandboxCallbackBridgeQueueClient = {
      ...base,
      listJsonFiles: async (dir) => {
        listCalls += 1;
        // The first poll (the loop) never resolves — a stalled channel that the
        // per-iteration timeout does not catch soon enough. Later calls (the
        // watchdog's failPendingRequests) resolve, so it enumerates and 503s.
        if (listCalls === 1) {
          return await new Promise<string[]>(() => {});
        }
        return await base.listJsonFiles(dir);
      },
    };

    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      // The per-iteration timeout is far larger than the watchdog threshold, so
      // the watchdog — not the per-iteration timeout — is the mechanism proven.
      iterationTimeoutMs: 400,
      watchdogTimeoutMs: 50,
      runtimeSpan,
      authorizeRequest: async () => null,
      handleRequest: async () => ({ status: 200, body: "ok" }),
    });

    const responseFile = await waitForJsonFile(directories.responsesDir, 3_000);
    const responseBody = await readFile(path.posix.join(directories.responsesDir, responseFile), "utf8");
    expect(JSON.parse(responseBody).status).toBe(503);
    expect(workerErrors.some((message) => message.includes("no successful poll iteration"))).toBe(true);

    await worker.stop({ drainTimeoutMs: 400 });
  });

  it("processes a fast request with no false-positive timeout and no run-level error", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-fast-"));
    cleanupDirs.push(rootDir);

    const queueDir = path.posix.join(rootDir, "queue");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    await mkdir(directories.requestsDir, { recursive: true });
    await publishBridgeRequest(path.posix.join(directories.requestsDir, "req-ok.json"), bridgeRequestJson("req-ok"));

    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();
    const processed: string[] = [];

    const worker = await startSandboxCallbackBridgeWorker({
      client: createFileSystemSandboxCallbackBridgeQueueClient(),
      queueDir,
      iterationTimeoutMs: 200,
      watchdogTimeoutMs: 1_000,
      runtimeSpan,
      authorizeRequest: async () => null,
      handleRequest: async (request) => {
        processed.push(request.id);
        return { status: 200, body: request.id };
      },
    });

    const responseFile = await waitForJsonFile(directories.responsesDir, 3_000);
    const responseBody = await readFile(path.posix.join(directories.responsesDir, responseFile), "utf8");
    expect(JSON.parse(responseBody).status).toBe(200);
    expect(JSON.parse(responseBody).body).toBe("req-ok");

    // Let several idle poll iterations and watchdog checks pass. A healthy idle
    // loop must never trip the watchdog and never surface a run-level error.
    await new Promise((resolve) => setTimeout(resolve, 300));
    expect(processed).toEqual(["req-ok"]);
    expect(workerErrors).toEqual([]);

    await worker.stop({ drainTimeoutMs: 50 });
  });

  async function prepareGatewayFixture(prefix: string) {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), prefix));
    cleanupDirs.push(rootDir);
    const localWorkspaceDir = path.join(rootDir, "local-workspace");
    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    await mkdir(localWorkspaceDir, { recursive: true });
    await mkdir(remoteWorkspaceDir, { recursive: true });
    await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge test\n", "utf8");

    const runner = createExecRunner();
    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const prepared = await prepareCommandManagedRuntime(queueRuntimeInput(runner, remoteWorkspaceDir, localWorkspaceDir, bridgeAsset));
    const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
    return {
      runner,
      remoteWorkspaceDir,
      assetRemoteDir: prepared.assetDirs.bridge,
      queueDir,
      directories: sandboxCallbackBridgeDirectories(queueDir),
      bridgeToken: createSandboxCallbackBridgeToken(),
    };
  }

  it("cleans up a timed-out request file and keeps serving after the host recovers", async () => {
    const fixture = await prepareGatewayFixture("paperclip-bridge-timeout-clean-");

    const bridge = await startSandboxCallbackBridgeServer({
      runner: fixture.runner,
      remoteCwd: fixture.remoteWorkspaceDir,
      assetRemoteDir: fixture.assetRemoteDir,
      queueDir: fixture.queueDir,
      bridgeToken: fixture.bridgeToken,
      timeoutMs: 30_000,
      responseTimeoutMs: 600,
      pollIntervalMs: 50,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    // No worker runs, so the request times out at the gateway.
    const timedOut = await fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: { authorization: `Bearer ${fixture.bridgeToken}` },
    });
    expect(timedOut.status).toBe(502);
    await expect(timedOut.json()).resolves.toMatchObject({
      error: expect.stringContaining("Timed out"),
    });

    // The gateway cleaned its own request file, so nothing counts toward the
    // queue-depth cap after the caller gave up.
    const leftover = (await readdir(fixture.directories.requestsDir).catch(() => [])).filter((name) =>
      name.endsWith(".json"),
    );
    expect(leftover).toEqual([]);

    // A worker that comes up afterwards serves the next request normally —
    // the timeout neither wedged nor killed the gateway.
    const worker = await startSandboxCallbackBridgeWorker({
      client: createFileSystemSandboxCallbackBridgeQueueClient(),
      queueDir: fixture.queueDir,
      authorizeRequest: async () => null,
      handleRequest: async () => ({
        status: 200,
        headers: { "content-type": "application/json" },
        body: JSON.stringify({ ok: true }),
      }),
    });
    cleanupFns.push(async () => {
      await worker.stop();
    });

    const recovered = await fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: { authorization: `Bearer ${fixture.bridgeToken}` },
    });
    expect(recovered.status).toBe(200);
    await expect(recovered.json()).resolves.toMatchObject({ ok: true });
  }, 30_000);

  it("sweeps stale request files before rejecting at the queue-depth cap", async () => {
    const fixture = await prepareGatewayFixture("paperclip-bridge-stale-sweep-");

    const bridge = await startSandboxCallbackBridgeServer({
      runner: fixture.runner,
      remoteCwd: fixture.remoteWorkspaceDir,
      assetRemoteDir: fixture.assetRemoteDir,
      queueDir: fixture.queueDir,
      bridgeToken: fixture.bridgeToken,
      timeoutMs: 30_000,
      responseTimeoutMs: 500,
      pollIntervalMs: 50,
      maxQueueDepth: 1,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    // Plant an orphaned request file (a killed caller, or a previous gateway
    // process's leftover) and backdate it beyond the response deadline.
    const orphanPath = path.join(fixture.directories.requestsDir, "orphan.json");
    await writeFile(orphanPath, bridgeRequestJson("orphan"), "utf8");
    const staleTime = new Date(Date.now() - 60_000);
    await utimes(orphanPath, staleTime, staleTime);

    // The queue sits at the cap, but the only entry is stale: the gateway must
    // sweep it and admit the request instead of answering 503. With no worker
    // the admitted request then times out (502) — proof it entered the queue.
    const response = await fetch(`${bridge.baseUrl}/api/agents/me`, {
      headers: { authorization: `Bearer ${fixture.bridgeToken}` },
    });
    expect(response.status).toBe(502);
    const body = (await response.json()) as { error?: string };
    expect(body.error ?? "").not.toContain("queue is full");

    const leftover = (await readdir(fixture.directories.requestsDir).catch(() => [])).filter((name) =>
      name.endsWith(".json"),
    );
    expect(leftover).toEqual([]);
  }, 30_000);

  it("skips a request file that vanished before the read instead of escalating", async () => {
    // The gateway deletes a request file when its caller stops waiting. The
    // worker's read then races the deletion; a vanished file must be a quiet
    // skip, not a worker failure with a recovery pass.
    const queueDir = "/virtual-bridge/vanished";
    let listCalls = 0;
    const handled: string[] = [];
    const responseStatuses: number[] = [];
    const { runtimeSpan, workerErrors } = createWorkerErrorCapture();

    const client: SandboxCallbackBridgeQueueClient = {
      makeDir: async () => {},
      makeDirs: async () => {},
      listJsonFiles: async () => {
        listCalls += 1;
        return listCalls === 1 ? ["ghost.json"] : [];
      },
      readTextFile: async () => {
        throw new Error("cat: ghost.json: No such file or directory");
      },
      writeTextFile: async () => {},
      writeResponseFile: async (_remotePath, body) => {
        responseStatuses.push((JSON.parse(body.trim()) as { status: number }).status);
        return { wrote: true };
      },
      rename: async () => {},
      remove: async () => {},
    };

    const worker = await startSandboxCallbackBridgeWorker({
      client,
      queueDir,
      runtimeSpan,
      authorizeRequest: async () => null,
      handleRequest: async (request) => {
        handled.push(request.id);
        return { status: 200, body: "ok" };
      },
    });

    await new Promise((resolve) => setTimeout(resolve, 250));
    await worker.stop({ drainTimeoutMs: 50 });

    expect(handled).toEqual([]);
    expect(responseStatuses).toEqual([]);
    expect(workerErrors).toEqual([]);
  });

  it("embeds crash handlers and queue hygiene in the generated gateway source", () => {
    // A crashed gateway is a dead loopback port for the rest of the run, so
    // the generated source must keep its crash handlers and its stale-queue
    // sweep. The readiness gate matters too: survival applies only after the
    // gateway is adoptable, so a startup fault still fails fast. This pins
    // their presence; the behavior is proven above and below.
    const source = getSandboxCallbackBridgeServerSource();
    expect(source).toContain('process.on("uncaughtException"');
    expect(source).toContain('process.on("unhandledRejection"');
    expect(source).toContain("gatewayReady");
    expect(source).toContain("sweepStaleRequests");
  });

  it("exits fast when the gateway cannot bind its port instead of lingering un-ready", async () => {
    // Before readiness, the crash handlers must not keep the process alive: a
    // failed bind means the gateway can never serve, and surviving would only
    // leave an un-ready zombie while the host waits out its readiness poll.
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-bind-fail-"));
    cleanupDirs.push(rootDir);
    const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
    await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
    const queueDir = path.join(rootDir, "queue");
    await mkdir(queueDir, { recursive: true });

    const blocker = createServer();
    await new Promise<void>((resolve) => {
      blocker.listen(0, "127.0.0.1", resolve);
    });
    cleanupFns.push(
      () =>
        new Promise<void>((resolve) => {
          blocker.close(() => resolve());
        }),
    );
    const blockedPort = (blocker.address() as { port: number }).port;

    const child = spawn(process.execPath, [entrypoint], {
      env: {
        ...process.env,
        PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir,
        PAPERCLIP_BRIDGE_TOKEN: "test-token",
        PAPERCLIP_BRIDGE_PORT: String(blockedPort),
      },
      stdio: ["ignore", "ignore", "pipe"],
    });
    let stderr = "";
    child.stderr.on("data", (chunk) => {
      stderr += chunk;
    });
    const exitCode = await new Promise<number | null>((resolve) => {
      child.on("close", resolve);
    });

    expect(exitCode).toBe(1);
    expect(stderr).toContain("[paperclip-bridge] server error");
    expect(stderr).toContain("EADDRINUSE");
  }, 15_000);

  it("exits nonzero for the retired duplex_v1 mode instead of starting the queue gateway", async () => {
    // The closed mode allowlist rejects `duplex_v1` before the queue-directory
    // check, so a stale `duplex_v1` launch environment fails startup instead
    // of silently falling through to the queue gateway.
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-mode-duplex-"));
    cleanupDirs.push(rootDir);
    const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
    await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
    const queueDir = path.join(rootDir, "queue");
    await mkdir(queueDir, { recursive: true });

    const child = spawn(process.execPath, [entrypoint], {
      env: {
        ...process.env,
        PAPERCLIP_API_BRIDGE_MODE: "duplex_v1",
        PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir,
        PAPERCLIP_BRIDGE_TOKEN: "test-token",
        PAPERCLIP_BRIDGE_PORT: "0",
      },
      stdio: ["ignore", "ignore", "pipe"],
    });
    let stderr = "";
    child.stderr.on("data", (chunk) => {
      stderr += chunk;
    });
    const exitCode = await new Promise<number | null>((resolve) => {
      child.on("close", resolve);
    });

    expect(exitCode).not.toBe(0);
    expect(stderr).toContain("Unsupported PAPERCLIP_API_BRIDGE_MODE: duplex_v1");
  }, 15_000);

  it("exits nonzero for an unknown bridge mode instead of starting the queue gateway", async () => {
    // The closed mode allowlist rejects every value it does not name, not
    // only the retired duplex transport.
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-mode-unknown-"));
    cleanupDirs.push(rootDir);
    const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
    await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
    const queueDir = path.join(rootDir, "queue");
    await mkdir(queueDir, { recursive: true });

    const child = spawn(process.execPath, [entrypoint], {
      env: {
        ...process.env,
        PAPERCLIP_API_BRIDGE_MODE: "totally_unknown_mode",
        PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir,
        PAPERCLIP_BRIDGE_TOKEN: "test-token",
        PAPERCLIP_BRIDGE_PORT: "0",
      },
      stdio: ["ignore", "ignore", "pipe"],
    });
    let stderr = "";
    child.stderr.on("data", (chunk) => {
      stderr += chunk;
    });
    const exitCode = await new Promise<number | null>((resolve) => {
      child.on("close", resolve);
    });

    expect(exitCode).not.toBe(0);
    expect(stderr).toContain("Unsupported PAPERCLIP_API_BRIDGE_MODE: totally_unknown_mode");
  }, 15_000);

  it("test_http2_gateway_writes_no_frame_between_ready_and_the_preface", async () => {
    // Spawn the real generated gateway in http2_v1 mode and read its raw
    // stdout bytes. The only frame-codec write on this path is the READY
    // line; the very next bytes must be the HTTP/2 client connection preface
    // with nothing in between, because the gateway hands stdout to the
    // HTTP/2 client immediately after it writes READY and starts no
    // heartbeat timer and writes no envelope frame on this path.
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-http2-gateway-"));
    cleanupDirs.push(rootDir);
    const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
    await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");

    const probe = createServer();
    const assignedPort = await new Promise<number>((resolve, reject) => {
      probe.once("error", reject);
      probe.listen(0, "127.0.0.1", () => {
        const address = probe.address();
        if (!address || typeof address === "string") {
          reject(new Error("Could not reserve a loopback port for the test."));
          return;
        }
        probe.close(() => resolve(address.port));
      });
    });

    const nonce = "test-nonce-http2";
    const child = spawn(process.execPath, [entrypoint], {
      env: {
        ...process.env,
        PAPERCLIP_API_BRIDGE_MODE: "http2_v1",
        PAPERCLIP_BRIDGE_TOKEN: "test-token",
        PAPERCLIP_BRIDGE_PORT: String(assignedPort),
        PAPERCLIP_BRIDGE_NONCE: nonce,
      },
      stdio: ["pipe", "pipe", "pipe"],
    });
    cleanupFns.push(async () => {
      child.kill();
    });
    let stderr = "";
    child.stderr.on("data", (chunk: Buffer) => {
      stderr += chunk.toString("utf8");
    });

    const chunks: Buffer[] = [];
    const preface = Buffer.from("505249202a20485454502f322e300d0a0d0a534d0d0a0d0a", "hex");
    const firstBytes = await new Promise<Buffer>((resolve, reject) => {
      const timer = setTimeout(
        () => reject(new Error("Timed out waiting for the http2 gateway stdout. stderr: " + stderr)),
        5000,
      );
      child.stdout.on("data", (chunk: Buffer) => {
        chunks.push(chunk);
        const total = Buffer.concat(chunks);
        const newlineIndex = total.indexOf(0x0a);
        if (newlineIndex !== -1 && total.length >= newlineIndex + 1 + preface.length) {
          clearTimeout(timer);
          resolve(total);
        }
      });
      child.once("error", reject);
      child.once("exit", (code) => {
        clearTimeout(timer);
        reject(new Error("The http2 gateway exited early with code " + String(code) + ". stderr: " + stderr));
      });
    });

    const newlineIndex = firstBytes.indexOf(0x0a);
    expect(newlineIndex).toBeGreaterThan(0);
    // Assert the exact UTF-8 bytes, not a parsed-and-matched object.
    // `JSON.stringify` writes keys in the object-literal insertion order, so a
    // reordered or reformatted call site at the gateway's one `writeFrame` call
    // would change the bytes on the wire without failing a looser assertion.
    const readyLine = firstBytes.subarray(0, newlineIndex).toString("utf8");
    expect(readyLine).toBe(`{"version":2,"type":"ready","nonce":"${nonce}"}`);
    expect(encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "ready", nonce })).toBe(`${readyLine}\n`);
    const afterReady = firstBytes.subarray(newlineIndex + 1, newlineIndex + 1 + preface.length);
    expect(afterReady).toEqual(preface);
  }, 15_000);

  /**
   * Spawn the real generated gateway in `http2_v1` mode, then bind the real
   * host-side `createHttp2BridgeServer` to its stdio. The gateway writes one
   * READY line before it hands stdout to its HTTP/2 client, so this helper
   * strips that line first and feeds the host only the raw HTTP/2 bytes that
   * follow. A test then drives the gateway with ordinary HTTP/1.1 requests
   * against its loopback port and inspects the exact bytes `forwardRequest`
   * receives, proving the send path carries no intermediate string.
   */
  async function startHttp2GatewayForTest(options: {
    bridgeToken: string;
    maxBodyBytes?: number;
    holdProbeOpen?: () => boolean;
    deferStderrUntilExit?: boolean;
    forwardRequest: (request: Http2BridgeForwardRequest) => Promise<Http2BridgeForwardResult>;
  }, attempt = 0): Promise<{ baseUrl: string; stop: () => Promise<void> }> {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-http2-test-"));
    cleanupDirs.push(rootDir);
    const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
    await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");

    const probe = createServer();
    const assignedPort = await new Promise<number>((resolve, reject) => {
      probe.once("error", reject);
      probe.listen(0, "127.0.0.1", () => {
        const address = probe.address();
        if (!address || typeof address === "string") {
          reject(new Error("Could not reserve a loopback port for the test."));
          return;
        }
        if (options.holdProbeOpen?.()) {
          cleanupFns.push(() => new Promise<void>((resolve) => probe.close(() => resolve())));
          resolve(address.port);
        } else {
          probe.close(() => resolve(address.port));
        }
      });
    });

    const child = spawn(process.execPath, [entrypoint], {
      env: {
        ...process.env,
        PAPERCLIP_API_BRIDGE_MODE: "http2_v1",
        PAPERCLIP_BRIDGE_TOKEN: options.bridgeToken,
        PAPERCLIP_BRIDGE_PORT: String(assignedPort),
        PAPERCLIP_BRIDGE_NONCE: "test-nonce",
        ...(options.maxBodyBytes != null
          ? { PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(options.maxBodyBytes) }
          : {}),
      },
      stdio: ["pipe", "pipe", "pipe"],
    });
    let stderr = "";
    child.stderr.on("data", (chunk: Buffer) => {
      stderr += chunk.toString("utf8");
    });

    if (options.deferStderrUntilExit) {
      const resume = child.stderr.resume.bind(child.stderr);
      child.stderr.pause();
      child.stderr.resume = () => child.stderr;
      child.once("exit", () => setImmediate(() => {
        child.stderr.resume = resume;
        resume();
      }));
    }

    const dataListeners: Array<(chunk: Uint8Array) => void> = [];
    const pending: Buffer[] = [];
    let dispatchStarted = false;
    let sawReadyLine = false;
    let readyBuffer = Buffer.alloc(0);
    let resolveReady!: () => void;
    let rejectReady!: (error: Error) => void;
    const readyPromise = new Promise<void>((resolve, reject) => {
      resolveReady = resolve;
      rejectReady = reject;
    });
    const readyTimer = setTimeout(
      () => rejectReady(new Error("Timed out waiting for the http2 gateway READY line. stderr: " + stderr)),
      5000,
    );
    const deliver = (chunk: Buffer) => {
      if (dispatchStarted) {
        for (const listener of dataListeners) listener(chunk);
      } else {
        pending.push(chunk);
      }
    };
    child.stdout.on("data", (chunk: Buffer) => {
      if (!sawReadyLine) {
        readyBuffer = Buffer.concat([readyBuffer, chunk]);
        const newlineIndex = readyBuffer.indexOf(0x0a);
        if (newlineIndex === -1) return;
        sawReadyLine = true;
        clearTimeout(readyTimer);
        const rest = readyBuffer.subarray(newlineIndex + 1);
        readyBuffer = Buffer.alloc(0);
        resolveReady();
        if (rest.length > 0) deliver(rest);
        return;
      }
      deliver(chunk);
    });
    child.once("close", (code) => {
      clearTimeout(readyTimer);
      if (!sawReadyLine) {
        rejectReady(new Error("The http2 gateway exited early with code " + String(code) + ". stderr: " + stderr));
      }
    });

    const channel: CommandManagedDuplexChannel = {
      write: (data) => {
        child.stdin.write(Buffer.from(data));
      },
      onData: (listener) => {
        dataListeners.push(listener);
      },
      onExit: (listener) => {
        child.once("exit", (code) => listener({ exitCode: code }));
      },
      stop: () => {
        child.kill();
      },
      close: async () => {
        child.stdin.end();
      },
    };

    try {
      await readyPromise;
    } catch (error) {
      child.kill();
      if (attempt < 2 && child.exitCode === 1
        && stderr.includes("http2 gateway could not bind port") && stderr.includes("EADDRINUSE")) {
        return startHttp2GatewayForTest(options, attempt + 1);
      }
      throw error;
    }

    const handle = createHttp2BridgeServer({
      bridgeToken: options.bridgeToken,
      forwardRequest: options.forwardRequest,
    });
    const boundDuplex = handle.bindChannel(channel);
    dispatchStarted = true;
    const buffered = pending.splice(0);
    for (const chunk of buffered) {
      for (const listener of dataListeners) listener(chunk);
    }

    // Destroy the bound duplex directly instead of `handle.close()`. The
    // duplex here wraps a spawned process's raw stdio, not a real socket,
    // and `close()` waits on a graceful HTTP/2 GOAWAY exchange that never
    // settles over this transport. A direct destroy ends the session at
    // once, which is correct for test teardown.
    const stop = async () => {
      boundDuplex.destroy();
      child.kill();
    };
    cleanupFns.push(stop);

    return { baseUrl: `http://127.0.0.1:${assignedPort}`, stop };
  }

  it("forwards the exact request body bytes to the HTTP/2 host handler, including a non-ASCII character", async () => {
    const bridgeToken = createSandboxCallbackBridgeToken();
    const seenBodies: Buffer[] = [];
    const gateway = await startHttp2GatewayForTest({
      bridgeToken,
      forwardRequest: async (request) => {
        seenBodies.push(request.body);
        return {
          status: 200,
          headers: { "content-type": "application/json" },
          body: Buffer.from(JSON.stringify({ ok: true }), "utf8"),
        };
      },
    });

    // "café" holds one multi-byte UTF-8 character. A body-to-string-to-body
    // round trip still reproduces this text correctly, so the byte-for-byte
    // comparison below is the real proof: it fails if any re-encoding step
    // runs, even one that happens to preserve valid UTF-8 text.
    const bodyText = JSON.stringify({ note: "café" });
    const bodyBytes = Buffer.from(bodyText, "utf8");
    const response = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, {
      method: "POST",
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "content-type": "application/json",
      },
      body: bodyBytes,
    });
    expect(response.status).toBe(200);
    expect(seenBodies).toHaveLength(1);
    expect(seenBodies[0]?.equals(bodyBytes)).toBe(true);
  }, 15_000);

  it("forwards malformed UTF-8 bytes to the HTTP/2 host handler unchanged", async () => {
    const bridgeToken = createSandboxCallbackBridgeToken();
    const seenBodies: Buffer[] = [];
    // Byte 0xC3 opens a two-byte UTF-8 sequence; 0x28 is not a valid
    // continuation byte, so this body is not valid UTF-8. The gateway does
    // not decode or validate the body, so these exact bytes must still
    // arrive at the host handler unchanged, and the same bytes must return
    // to the caller unchanged. The host answers with a non-JSON content
    // type, so the round trip proves the gateway applies no format-specific
    // handling on the response leg either.
    const malformedBytes = Buffer.from([0x7b, 0x22, 0x61, 0x22, 0x3a, 0xc3, 0x28, 0x7d]);
    const gateway = await startHttp2GatewayForTest({
      bridgeToken,
      forwardRequest: async (request) => {
        seenBodies.push(request.body);
        return { status: 200, headers: { "content-type": "application/octet-stream" }, body: malformedBytes };
      },
    });

    const response = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, {
      method: "POST",
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "content-type": "application/json",
      },
      body: malformedBytes,
    });
    expect(response.status).toBe(200);
    expect(seenBodies).toHaveLength(1);
    expect(seenBodies[0]?.equals(malformedBytes)).toBe(true);
    const responseBytes = Buffer.from(await response.arrayBuffer());
    expect(responseBytes.equals(malformedBytes)).toBe(true);
  }, 15_000);

  it("rejects a request body over maxBodyBytes on the HTTP/2 path before it forwards a byte", async () => {
    const bridgeToken = createSandboxCallbackBridgeToken();
    const maxBodyBytes = 32;
    let forwardCalls = 0;
    const gateway = await startHttp2GatewayForTest({
      bridgeToken,
      maxBodyBytes,
      forwardRequest: async () => {
        forwardCalls += 1;
        return { status: 200, headers: {}, body: Buffer.alloc(0) };
      },
    });

    const oversizeBody = Buffer.alloc(maxBodyBytes + 1, 0x41);
    const response = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, {
      method: "POST",
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "content-type": "application/json",
      },
      body: oversizeBody,
    });
    expect(response.status).toBe(502);
    await expect(response.json()).resolves.toMatchObject({
      error: "Bridge request body exceeded the configured size limit.",
    });
    expect(forwardCalls).toBe(0);
  }, 15_000);

  /** A resolvable gate a test can await, then release on its own schedule. */
  function createGate(): { reached: Promise<void>; reach: () => void } {
    let reach!: () => void;
    const reached = new Promise<void>((resolve) => {
      reach = resolve;
    });
    return { reached, reach };
  }

  it("denies a body once concurrent reservations reach the gateway's own process ledger ceiling, then admits again once they release", async () => {
    // The per-body maxBodyBytes cap alone does not bound how many bodies
    // this gateway process holds in memory at once. The generated gateway's
    // own process ledger gives it a second, independent ceiling: 4
    // concurrent max-size bodies, each counted twice (the retained chunk
    // array and the concatenated copy), exactly fills maxBodyBytes * 8. This
    // drives 4 concurrent requests to that exact ceiling, held open by a
    // forward call that does not return, then proves a 5th is denied and a
    // later one succeeds once the 4 held requests release their bytes.
    const bridgeToken = createSandboxCallbackBridgeToken();
    const maxBodyBytes = 400;
    const holdBody = Buffer.alloc(maxBodyBytes, 0x42);
    const HOLD_COUNT = 4;
    const reachedGates = Array.from({ length: HOLD_COUNT }, () => createGate());
    const releaseGates = Array.from({ length: HOLD_COUNT }, () => createGate());
    let forwardCalls = 0;
    const gateway = await startHttp2GatewayForTest({
      bridgeToken,
      maxBodyBytes,
      forwardRequest: async () => {
        const callIndex = forwardCalls;
        forwardCalls += 1;
        if (callIndex < HOLD_COUNT) {
          reachedGates[callIndex]!.reach();
          await releaseGates[callIndex]!.reached;
        }
        return { status: 200, headers: {}, body: Buffer.alloc(0) };
      },
    });

    const postHoldBody = () =>
      fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, {
        method: "POST",
        headers: { authorization: `Bearer ${bridgeToken}`, "content-type": "application/json" },
        body: holdBody,
      });

    const holdResponses = Promise.all(Array.from({ length: HOLD_COUNT }, () => postHoldBody()));
    // Each held request's own forward call started, which only happens
    // after readBodyBytes already reserved that request's bytes, so all 4
    // reservations are live once every gate below resolves.
    await Promise.all(reachedGates.map((gate) => gate.reached));

    const deniedResponse = await postHoldBody();
    expect(deniedResponse.status).toBe(503);
    await expect(deniedResponse.json()).resolves.toMatchObject({
      error: "The bridge gateway process reached its reserved body byte ceiling. Retry later.",
    });
    // The denial happened during the body read, before this stream's own
    // forward call ever ran.
    expect(forwardCalls).toBe(HOLD_COUNT);

    releaseGates.forEach((gate) => gate.reach());
    const responses = await holdResponses;
    for (const response of responses) {
      expect(response.status).toBe(200);
    }

    // Every held reservation released once its own forward call settled: a
    // fresh request at the same size now succeeds again.
    const recoveredResponse = await postHoldBody();
    expect(recoveredResponse.status).toBe(200);
  }, 15_000);

  it("maps an indeterminate host outcome to a non-retryable 409 on the HTTP/2 gateway for an unsafe method", async () => {
    // The host answers a mutating request's response-body capacity denial with a
    // 504 and the indeterminate outcome header, exactly like an aborted in-flight
    // forward: the host may have already committed the mutation, so the status
    // must not be retryable. The HTTP/2 gateway must map that 504 to a 409, the
    // same map the file gateway already applies, so a standard retry policy does
    // not repeat the mutation. The outcome header and body must survive the map.
    const bridgeToken = createSandboxCallbackBridgeToken();
    const indeterminateBody = JSON.stringify({ error: "response body over limit", outcome: "indeterminate" });
    const gateway = await startHttp2GatewayForTest({
      bridgeToken,
      forwardRequest: async () => ({
        status: 504,
        headers: { "content-type": "application/json", "x-paperclip-bridge-outcome": "indeterminate" },
        body: Buffer.from(indeterminateBody, "utf8"),
      }),
    });

    const response = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, {
      method: "POST",
      headers: { authorization: `Bearer ${bridgeToken}`, "content-type": "application/json" },
      body: JSON.stringify({ note: "test" }),
    });

    expect(response.status).toBe(409);
    expect(response.status).toBeLessThan(500);
    expect(response.headers.get("x-paperclip-bridge-outcome")).toBe("indeterminate");
    const responseBytes = Buffer.from(await response.arrayBuffer());
    expect(responseBytes.toString("utf8")).toBe(indeterminateBody);
  }, 15_000);

  it("passes through a safe method's retryable 503 on the HTTP/2 gateway unchanged", async () => {
    // A safe method's response-body capacity denial carries no indeterminate
    // outcome header, so the gateway must keep it retryable: the map above must
    // not fire on a plain 503.
    const bridgeToken = createSandboxCallbackBridgeToken();
    const gateway = await startHttp2GatewayForTest({
      bridgeToken,
      forwardRequest: async () => ({
        status: 503,
        headers: { "content-type": "application/json" },
        body: Buffer.from(JSON.stringify({ error: "response body over limit" }), "utf8"),
      }),
    });

    const response = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, {
      method: "GET",
      headers: { authorization: `Bearer ${bridgeToken}` },
    });

    expect(response.status).toBe(503);
    expect(response.headers.get("x-paperclip-bridge-outcome")).toBeNull();
  }, 15_000);

  async function startQueueGatewayForFileTest(options: {
    maxBodyBytes: number;
    client?: SandboxCallbackBridgeQueueClient;
    handleRequest: Parameters<typeof startSandboxCallbackBridgeWorker>[0]["handleRequest"];
  }) {
    const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-file-codec-"));
    cleanupDirs.push(root);
    const asset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(asset.cleanup);
    const queueDir = path.join(root, "queue");
    const bridgeToken = createSandboxCallbackBridgeToken();
    const worker = await startSandboxCallbackBridgeWorker({
      client: options.client ?? createFileSystemSandboxCallbackBridgeQueueClient(), queueDir,
      maxBodyBytes: options.maxBodyBytes, handleRequest: options.handleRequest, pollIntervalMs: 10,
    });
    cleanupFns.push(() => worker.stop());
    const gateway = await startSandboxCallbackBridgeServer({
      runner: createExecRunner(), remoteCwd: root, assetRemoteDir: asset.localDir,
      queueDir, bridgeToken, maxBodyBytes: options.maxBodyBytes, pollIntervalMs: 10,
    });
    cleanupFns.push(() => gateway.stop());
    return { ...gateway, bridgeToken, queueDir };
  }

  it.each(["queue", "http2"])("preserves multipart bytes at the configured limit on %s and rejects overflow before forwarding", async mode => {
    const maxBodyBytes = 1024;
    const bytes = Buffer.alloc(maxBodyBytes, 0xff);
    const handled = vi.fn(async (request: { body?: string | Buffer }) => ({
      status: 200, headers: { "content-type": "application/octet-stream" }, body: Buffer.from(request.body ?? ""),
    }));
    const bridgeToken = createSandboxCallbackBridgeToken();
    const gateway = mode === "queue"
      ? await startQueueGatewayForFileTest({ maxBodyBytes, handleRequest: handled })
      : { ...await startHttp2GatewayForTest({ bridgeToken, maxBodyBytes, forwardRequest: handled }), bridgeToken };
    const post = (body: Buffer) => fetch(`${gateway.baseUrl}/api/companies/c/issues/i/attachments`, {
      method: "POST", headers: { authorization: `Bearer ${gateway.bridgeToken}`, "content-type": "multipart/form-data; boundary=exact-boundary" }, body: new Uint8Array(body),
    });
    const response = await post(bytes);
    expect(response.status).toBe(200);
    expect(Buffer.from(await response.arrayBuffer())).toEqual(bytes);
    expect(handled).toHaveBeenCalledTimes(1);
    expect(handled.mock.calls[0][0]).toMatchObject({ headers: { "content-type": "multipart/form-data; boundary=exact-boundary" } });
    const overflow = await post(Buffer.alloc(maxBodyBytes + 1));
    expect(overflow.status).toBeGreaterThanOrEqual(400);
    expect(handled).toHaveBeenCalledTimes(1);
  });

  it("rejects malformed queue encodings without forwarding a mutation", async () => {
    const handled = vi.fn(async () => ({ status: 200, body: "ok" }));
    const gateway = await startQueueGatewayForFileTest({ maxBodyBytes: 1024, handleRequest: handled });
    const directories = sandboxCallbackBridgeDirectories(gateway.queueDir);
    for (const [id, bodyEncoding, body] of [["bad-base64", "base64", "YR=="], ["bad-encoding", "hex", "00"], ["too-big", "base64", Buffer.alloc(1025).toString("base64")]]) {
      await createFileSystemSandboxCallbackBridgeQueueClient().writeTextFile(path.join(directories.requestsDir, `${id}.json`), JSON.stringify({
        id, method: "POST", path: "/api/companies/c/issues/i/attachments", query: "", headers: {}, bodyEncoding, body,
      }));
      const responsePath = path.join(directories.responsesDir, `${id}.json`);
      await vi.waitFor(async () => expect(JSON.parse(await readFile(responsePath, "utf8")).status).toBe(400));
    }
    expect(handled).not.toHaveBeenCalled();
  });

  it("reports a corrupted upload response as indeterminate and never forwards twice", async () => {
    const client = createFileSystemSandboxCallbackBridgeQueueClient();
    const writeResponse = client.writeResponseFile!.bind(client);
    client.writeResponseFile = (destination, body, options) => writeResponse(destination,
      JSON.stringify({ ...JSON.parse(body), bodyEncoding: "base64", body: "invalid" }), options);
    const handled = vi.fn(async () => ({ status: 201, body: Buffer.from("saved") }));
    const gateway = await startQueueGatewayForFileTest({ client, maxBodyBytes: 1024, handleRequest: handled });
    const response = await fetch(`${gateway.baseUrl}/api/companies/c/issues/i/attachments`, {
      method: "POST", headers: { authorization: `Bearer ${gateway.bridgeToken}`, "content-type": "multipart/form-data; boundary=boundary" }, body: Buffer.from([0xff]),
    });
    expect(response.status).toBe(409);
    expect(response.headers.get("x-paperclip-bridge-outcome")).toBe("indeterminate");
    await expect(response.json()).resolves.toMatchObject({ retryable: false, outcome: "indeterminate" });
    expect(handled).toHaveBeenCalledTimes(1);
  });

  it("rejects a request body over maxBodyBytes on the queue path before it writes the queue file", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-queue-maxbody-"));
    cleanupDirs.push(rootDir);
    const localWorkspaceDir = path.join(rootDir, "local-workspace");
    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    await mkdir(localWorkspaceDir, { recursive: true });
    await mkdir(remoteWorkspaceDir, { recursive: true });
    await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge maxBodyBytes test\n", "utf8");

    const runner = createExecRunner();
    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const prepared = await prepareCommandManagedRuntime(queueRuntimeInput(runner, remoteWorkspaceDir, localWorkspaceDir, bridgeAsset));

    const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
    const directories = sandboxCallbackBridgeDirectories(queueDir);
    const bridgeToken = createSandboxCallbackBridgeToken();
    const maxBodyBytes = 32;

    const bridge = await startSandboxCallbackBridgeServer({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: prepared.assetDirs.bridge,
      queueDir,
      bridgeToken,
      timeoutMs: 30_000,
      maxBodyBytes,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    const oversizeBody = Buffer.alloc(maxBodyBytes + 1, 0x41);
    const response = await fetch(`${bridge.baseUrl}/api/issues/issue-1/comments`, {
      method: "POST",
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "content-type": "application/json",
      },
      body: oversizeBody,
    });
    expect(response.status).toBe(502);
    await expect(response.json()).resolves.toMatchObject({
      error: "Bridge request body exceeded the configured size limit.",
    });

    const requestFiles = await readdir(directories.requestsDir);
    expect(requestFiles.filter((name) => name.endsWith(".json"))).toHaveLength(0);
  });

  it("keeps the queue request payload's body field as a plain JSON string", async () => {
    const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-queue-body-shape-"));
    cleanupDirs.push(rootDir);
    const localWorkspaceDir = path.join(rootDir, "local-workspace");
    const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
    await mkdir(localWorkspaceDir, { recursive: true });
    await mkdir(remoteWorkspaceDir, { recursive: true });
    await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge body shape test\n", "utf8");

    const runner = createExecRunner();
    const bridgeAsset = await createSandboxCallbackBridgeAsset();
    cleanupFns.push(bridgeAsset.cleanup);
    const prepared = await prepareCommandManagedRuntime(queueRuntimeInput(runner, remoteWorkspaceDir, localWorkspaceDir, bridgeAsset));

    const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
    const bridgeToken = createSandboxCallbackBridgeToken();
    const requestBodyText = JSON.stringify({ note: "café" });

    const seenRequests: Array<{ body: string | Buffer }> = [];
    const worker = await startSandboxCallbackBridgeWorker({
      client: createFileSystemSandboxCallbackBridgeQueueClient(),
      queueDir,
      authorizeRequest: async () => null,
      handleRequest: async (request) => {
        seenRequests.push({ body: request.body });
        return { status: 200, headers: {}, body: JSON.stringify({ ok: true }) };
      },
    });
    cleanupFns.push(async () => {
      await worker.stop();
    });

    const bridge = await startSandboxCallbackBridgeServer({
      runner,
      remoteCwd: remoteWorkspaceDir,
      assetRemoteDir: prepared.assetDirs.bridge,
      queueDir,
      bridgeToken,
      timeoutMs: 30_000,
    });
    cleanupFns.push(async () => {
      await bridge.stop();
    });

    const response = await fetch(`${bridge.baseUrl}/api/issues/issue-1/comments`, {
      method: "POST",
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "content-type": "application/json",
      },
      body: requestBodyText,
    });
    expect(response.status).toBe(200);
    expect(seenRequests).toHaveLength(1);
    expect(typeof seenRequests[0]?.body).toBe("string");
    expect(seenRequests[0]?.body).toBe(requestBodyText);
  });

  it("does not retry unrelated HTTP/2 fixture startup failures", async () => {
    let attempts = 0;
    await expect(startHttp2GatewayForTest({
      bridgeToken: "",
      holdProbeOpen: () => { attempts++; return false; },
      forwardRequest: async () => { throw new Error("Unexpected request to an unstarted gateway"); },
    })).rejects.toThrow("PAPERCLIP_BRIDGE_TOKEN");
    expect(attempts).toBe(1);
  });

  it("limits HTTP/2 fixture retries when every assigned port is occupied", async () => {
    let attempts = 0;
    await expect(startHttp2GatewayForTest({
      bridgeToken: createSandboxCallbackBridgeToken(),
      holdProbeOpen: () => { attempts++; return true; },
      forwardRequest: async () => { throw new Error("Unexpected request to an unstarted gateway"); },
    })).rejects.toThrow("EADDRINUSE");
    expect(attempts).toBe(3);
  }, 15000);

  it.each([[false, false], [true, false], [true, true]])("accepts the bridge token in X-API-Key on the HTTP/2 path and strips the header (collision=%s, late stderr=%s)", async (portCollision, lateStderr) => {
    // Both generated gateways share one token reader, so the HTTP/2 transport
    // must accept "X-API-Key" exactly as the file-mode transport does. The
    // header allowlist must still strip the credential before the gateway
    // forwards the request to the host.
    const bridgeToken = createSandboxCallbackBridgeToken();
    const seenHeaders: Array<Record<string, string>> = [];
    let attempts = 0;
    const gateway = await startHttp2GatewayForTest({
      bridgeToken,
      holdProbeOpen: () => { attempts++; return portCollision && attempts === 1; },
      deferStderrUntilExit: lateStderr,
      forwardRequest: async (request) => {
        seenHeaders.push(request.headers);
        return {
          status: 200,
          headers: { "content-type": "application/json" },
          body: Buffer.from(JSON.stringify({ ok: true }), "utf8"),
        };
      },
    });

    expect(attempts).toBeGreaterThanOrEqual(portCollision ? 2 : 1);
    expect(attempts).toBeLessThanOrEqual(3);

    const response = await fetch(`${gateway.baseUrl}/api/agents/me`, {
      headers: {
        "x-api-key": bridgeToken,
        accept: "application/json",
      },
    });
    expect(response.status).toBe(200);
    await expect(response.json()).resolves.toMatchObject({ ok: true });
    expect(seenHeaders).toHaveLength(1);
    expect(seenHeaders[0]?.["x-api-key"]).toBeUndefined();

    const deniedResponse = await fetch(`${gateway.baseUrl}/api/agents/me`, {
      headers: {
        "x-api-key": "wrong-token",
      },
    });
    expect(deniedResponse.status).toBe(401);
    await expect(deniedResponse.json()).resolves.toMatchObject({
      error: "Invalid bridge token.",
    });
    // The rejected request never opened a stream to the host.
    expect(seenHeaders).toHaveLength(1);

    // A bearer token still works and still wins when both headers are present.
    const bearerResponse = await fetch(`${gateway.baseUrl}/api/agents/me`, {
      headers: {
        authorization: `Bearer ${bridgeToken}`,
        "x-api-key": "wrong-token",
      },
    });
    expect(bearerResponse.status).toBe(200);
    expect(seenHeaders).toHaveLength(2);
    expect(seenHeaders[1]?.authorization).toBeUndefined();
    expect(seenHeaders[1]?.["x-api-key"]).toBeUndefined();
  }, 15_000);
});

interface EmbeddedBridgeProcessBodyLedger {
  reserve(byteCount: number): boolean;
  release(byteCount: number): void;
  readonly reservedBytes: number;
}

// This describe block covers the zero-dependency process body-byte ledger
// every generated gateway embeds (`BRIDGE_PROCESS_BODY_LEDGER_SOURCE`),
// exercised directly with no spawned process involved, the same way the
// codec source in `execution-target-sandbox.test.ts` gets its own direct
// coverage.
describe("embedded sandbox gateway process body ledger", () => {
  it("reserves and releases bytes against its own ceiling, denying only once it is exceeded", () => {
    const ledgerFactory = new Function(
      `${getSandboxBridgeProcessBodyLedgerSource()}\nreturn createBridgeProcessBodyLedger;`,
    ) as unknown as () => (maxBytes: number) => EmbeddedBridgeProcessBodyLedger;
    const createBridgeProcessBodyLedger = ledgerFactory();
    const ledger = createBridgeProcessBodyLedger(100);

    expect(ledger.reserve(60)).toBe(true);
    expect(ledger.reservedBytes).toBe(60);
    // A denied reservation reserves nothing: the total stays exactly what
    // the first call reserved.
    expect(ledger.reserve(41)).toBe(false);
    expect(ledger.reservedBytes).toBe(60);
    expect(ledger.reserve(40)).toBe(true);
    expect(ledger.reservedBytes).toBe(100);

    ledger.release(60);
    expect(ledger.reservedBytes).toBe(40);
    // Room freed by the release admits a request the full ceiling would
    // have denied.
    expect(ledger.reserve(60)).toBe(true);
    expect(ledger.reservedBytes).toBe(100);
  });
});