Merge origin/master and renumber routine migration

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-25 19:47:14 +00:00
commit 8f70a8e125
374 files changed
+95243 -10084

No files matched your search

@@ -168,6 +168,11 @@ Guidelines:
- write from the user perspective
- keep highlights short and concrete
- spell out upgrade actions for breaking changes
- **write at full stable depth from the first pass**: the beta-keyed
draft ships verbatim as the stable's notes, so the previous stable's
file is the density bar the moment the draft is first written — never
leave it at generated-skeleton density for the soak. The skeleton's
nested PR summaries are raw material to rewrite, not a format to keep.
- **describe deltas, not repeats**: read the previous stable's notes
(`releases/v<last-stable>.md`) before writing. When they already
introduced a feature, this release's entry covers only what changed —
+19
View File
@@ -194,6 +194,25 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# PID 1 must be an init that reaps adopted orphans. With node there, the
# orphans agent runs leave behind are never wait()ed and pin as zombies
# until the cgroup pid limit is exhausted and every fork() in the
# container fails. Run against the pushed image rather than a local
# build: the step above is multi-arch with `push: true`, so nothing is
# loaded into the runner's daemon. The cloud variant is FROM production
# and inherits the same ENTRYPOINT, so checking this image covers both.
- name: Verify PID 1 reaps orphaned processes
env:
# Through the environment, not interpolated into the script body, so
# the tag text is data rather than shell.
IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
run: |
set -euo pipefail
image="$(printf '%s\n' "$IMAGE_TAGS" | head -n 1)"
test -n "$image"
echo "Verifying orphan reaping in $image"
docker run --rm -i --pull always "$image" sh -s < scripts/assert-orphan-reaping.sh
# The cloud variant carries built bundled plugins for managed deployments
# (see the `cloud` stage in the Dockerfile). It runs as its own job with no
# `needs:` on the stock publish above, so the two builds run in parallel and
+3
View File
@@ -280,6 +280,9 @@ jobs:
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Verify Paperclip Runner
run: pnpm --filter @paperclipai/paperclip-runner check:all
- name: Build
run: pnpm build
+59
View File
@@ -38,6 +38,65 @@ on:
type: string
jobs:
# The Docker smoke below can never exercise the background-service leg of
# onboarding: containers have no service manager, so v2026.824.0 shipped a
# service install that crash-looped on a missing shim while every
# golden-path check stayed green. Run the same published artifact directly
# on the runner VM's systemd and require the installed service to end up
# serving.
smoke_service:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 24
- name: Start a user systemd session
# The hosted runner has no login session for the runner user, so
# `systemctl --user` cannot reach a user manager until lingering
# starts one. Export the session address for the steps below.
run: |
sudo loginctl enable-linger "$(id -un)"
uid="$(id -u)"
for _ in $(seq 1 30); do
[[ -S "/run/user/$uid/bus" ]] && break
sleep 1
done
[[ -S "/run/user/$uid/bus" ]]
{
echo "XDG_RUNTIME_DIR=/run/user/$uid"
echo "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus"
} >> "$GITHUB_ENV"
- name: Onboard with the background service
env:
PAPERCLIPAI_VERSION: ${{ inputs.paperclip_version }}
DATA_DIR: ${{ runner.temp }}/service-smoke-data
SMOKE_CLEANUP: "false"
run: ./scripts/service-onboard-smoke.sh
- name: Capture service diagnostics
if: always()
run: |
{
systemctl --user --no-pager status paperclipai.service || true
journalctl --user -u paperclipai.service --no-pager || true
} > "$RUNNER_TEMP/paperclipai-service.log" 2>&1
- name: Upload service diagnostics
if: always()
uses: actions/upload-artifact@v7
with:
name: ${{ inputs.artifact_name }}-service
path: ${{ runner.temp }}/paperclipai-service.log
retention-days: 14
smoke:
runs-on: ubuntu-latest
timeout-minutes: 45
+3
View File
@@ -182,5 +182,8 @@ jobs:
- name: Install dependencies
run: pnpm install --no-frozen-lockfile
- name: Verify Paperclip Runner
run: pnpm --filter @paperclipai/paperclip-runner check:all
- name: Build
run: pnpm build
+6
View File
@@ -647,6 +647,9 @@ jobs:
timeout-minutes: 10
permissions:
contents: write
# gh pr view needs PR read for the skeleton's nested summaries;
# without it the enrichment silently degrades to bare subjects.
pull-requests: read
steps:
- name: Checkout repository
uses: actions/checkout@v7
@@ -661,6 +664,9 @@ jobs:
- name: Draft stable notes from the published beta
env:
# gh needs a token so the generator can nest each PR's summary
# under its subject line (best-effort thoroughness).
GH_TOKEN: ${{ github.token }}
BETA_VERSION: ${{ needs.publish_beta.outputs.beta_version }}
SOURCE_SHA: ${{ needs.select_beta.outputs.sha }}
run: |
+26
View File
@@ -91,6 +91,32 @@ When you are creating a plan file in the repository itself, new plan documents b
6. Attach inspectable generated artifacts.
When your task produces a user-inspectable deliverable file, follow the Paperclip skill's "Generated Artifacts and Work Products" workflow before final disposition. In this repo, prefer the self-contained skill helper at `skills/paperclip/scripts/paperclip-upload-artifact.sh` so the file is available through the Paperclip API, create/update an artifact work product when the file is the deliverable, link the uploaded artifact in the final issue comment, and then set status. Do not rely on local filesystem paths as the only access path. If an important file intentionally remains workspace-only, create/update a work product with `metadata.resourceRef.kind: "workspace_file"` and a workspace-relative path, then name that work product and path in the final comment. Treat browse/search as a fallback for recovering workspace files, not the preferred deliverable path. See `doc/AGENT-ARTIFACTS.md` for details and `.mp4`/`.webm` examples.
7. Name the three data paths correctly.
This repo has three separate data paths. Do not confuse them. Match a change to a path by its file path, not by the word "observability" or "telemetry" alone.
- **Telemetry** is the Paperclip first-party event system. It is opt-out and it sends data to a Paperclip endpoint by default. Its paths are:
- `packages/shared/src/telemetry/`
- the generated contract `packages/shared/src/telemetry/generated/paperclip-telemetry.ts`
- each caller of `packages/shared/src/telemetry/events.ts` or `packages/shared/src/telemetry/client.ts`
- **Observability** is the OpenTelemetry trace path. An operator must set an OTLP endpoint. Until an operator sets the endpoint, the tracer is a no-operation. Its paths are:
- `server/src/instrumentation.ts`
- `doc/observability.md`
- `packages/adapter-utils/src/duplex-observability.ts`
- `server/src/services/duplex-observability-recorder.ts`
- the span attributes in `packages/adapter-utils/src/acpx-engine/startup-timing.ts`
- **The run log** holds rows in the local `heartbeat_run_events` table. The data stays in the instance database. Its paths are:
- `doc/run-log-events.md`
- `packages/db/src/schema/heartbeat_run_events.ts`
- the append path `appendRunEvent` in `server/src/services/heartbeat.ts`
Apply a review level that matches the path:
- **Telemetry change (strict review).** The author updates the generated contract first. The author updates `packages/shared/src/telemetry/README.md` in the same pull request. The author requests a privacy review. Reason: a Telemetry event goes to a Paperclip endpoint by default, so a mistake sends data immediately.
- **Observability change (lighter review).** The operator endpoint gate stays in place. The no-operation behaviour stays when no endpoint is set. A privacy review is not necessary while the change stays inside the closed span-attribute allowlist.
- **Run-log change (no extra review).** A run-log change needs neither review level above, because the data stays in the instance database.
**Exclusion.** The word "observability" in a file such as `server/src/services/recovery-observability.ts` names a different concept. Apply this rule by path, not by word match.
## 6. Database Change Workflow
When changing data model:
+3 -1
View File
@@ -104,7 +104,9 @@ All tests must pass before a PR can be merged. Run them locally first and verify
### Telemetry Changes
If your change adds, removes, or modifies emitted telemetry events, update the [Telemetry Data Contract](packages/shared/src/telemetry/README.md) in the same PR. Keep clients emitting raw dimension values and avoid documenting or relying on private delivery details.
This repo has three separate data paths: Telemetry, Observability, and the run log. See rule 7 in `AGENTS.md` for the full definitions and the review level each path needs.
If your change adds, removes, or modifies emitted telemetry events, update the [Telemetry Data Contract](packages/shared/src/telemetry/README.md) in the same PR. Keep clients emitting raw dimension values and avoid documenting or relying on private delivery details. If your change adds, removes, or modifies an OpenTelemetry span or span attribute, keep the change inside the closed span-attribute allowlist in `packages/adapter-utils/src/acpx-engine/startup-timing.ts`. If your change adds or modifies a run-log event, update `doc/run-log-events.md` in the same PR.
### Paperclip Gates Must Pass
+10 -2
View File
@@ -3,7 +3,7 @@ FROM node:24-trixie-slim AS base
ARG USER_UID=1000
ARG USER_GID=1000
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates gosu curl gh git wget ripgrep python3 \
&& apt-get install -y --no-install-recommends ca-certificates gosu curl gh git wget ripgrep python3 tini \
&& rm -rf /var/lib/apt/lists/* \
&& corepack enable
@@ -24,6 +24,7 @@ COPY packages/adapter-utils/package.json packages/adapter-utils/
COPY packages/google-sheets-mcp-server/package.json packages/google-sheets-mcp-server/
COPY packages/kv-demo-mcp-server/package.json packages/kv-demo-mcp-server/
COPY packages/mcp-server/package.json packages/mcp-server/
COPY packages/paperclip-runner/package.json packages/paperclip-runner/
COPY packages/skills-catalog/package.json packages/skills-catalog/
COPY packages/tailscale-https-broker/package.json packages/tailscale-https-broker/
COPY packages/teams-catalog/package.json packages/teams-catalog/
@@ -118,7 +119,14 @@ ENV NODE_ENV=production \
EXPOSE 3100
ENTRYPOINT ["docker-entrypoint.sh"]
# tini, not node, is PID 1. The entrypoint ends in `exec`, so without an init
# node inherits PID 1 and never wait()s the orphans the kernel re-parents onto
# it -- agent runs spawn git/claude/esbuild/sh descendants that outlive their
# leader, so they pile up as permanent zombies (~79/h measured) until the
# cgroup pid limit is exhausted and *every* fork() in the container fails.
# tini reaps adopted orphans and forwards signals, so the exec chain below and
# graceful shutdown are unchanged. Mirrors docker/agent-runtime/Dockerfile.base.
ENTRYPOINT ["/usr/bin/tini", "--", "docker-entrypoint.sh"]
CMD ["node", "--import", "./server/node_modules/tsx/dist/loader.mjs", "server/dist/index.js"]
# Cloud image variant (build with `--target cloud`): the production image
+191 -3
View File
@@ -1,5 +1,30 @@
import { describe, expect, it, vi } from "vitest";
import { handleOnboardService } from "../onboard-service.js";
import {
handleOnboardService,
handoffToOnboardedService,
isInstallableReleaseVersion,
resolveOnboardServiceDashboardUrl,
shouldOfferForegroundStart,
} from "../onboard-service.js";
function dashboardConfig(overrides: {
host?: string;
port?: number;
baseUrlMode?: "auto" | "explicit";
publicBaseUrl?: string;
} = {}) {
return {
server: {
host: overrides.host ?? "127.0.0.1",
port: overrides.port ?? 3100,
},
auth: {
baseUrlMode: overrides.baseUrlMode ?? "auto",
disableSignUp: false,
...(overrides.publicBaseUrl ? { publicBaseUrl: overrides.publicBaseUrl } : {}),
},
};
}
function supportedDetection() {
return {
@@ -24,6 +49,7 @@ function supportedDetection() {
pid: 123,
})),
logs: vi.fn(async () => undefined),
installedExecutablePath: vi.fn(async () => null),
},
};
}
@@ -48,7 +74,11 @@ describe("onboard service policy", () => {
const installed = await handleOnboardService(
{ yes: true, installService: true },
{ detect: vi.fn(async () => detection), isInteractive: () => false },
{
detect: vi.fn(async () => detection),
isInteractive: () => false,
ensureServiceShim: vi.fn(async () => ({ ok: true, installedNow: false })),
},
);
expect(installed).toBe(true);
@@ -61,7 +91,12 @@ describe("onboard service policy", () => {
const installed = await handleOnboardService(
{},
{ detect: vi.fn(async () => detection), isInteractive: () => true, confirm },
{
detect: vi.fn(async () => detection),
isInteractive: () => true,
confirm,
ensureServiceShim: vi.fn(async () => ({ ok: true, installedNow: false })),
},
);
expect(confirm).toHaveBeenCalledOnce();
@@ -81,4 +116,157 @@ describe("onboard service policy", () => {
expect(detect).not.toHaveBeenCalled();
expect(info).not.toHaveBeenCalled();
});
it("materializes the managed shim before installing the service", async () => {
const detection = supportedDetection();
const success = vi.fn();
const ensureServiceShim = vi.fn(async () => ({ ok: true, installedNow: true }));
const installed = await handleOnboardService(
{ yes: true, installService: true },
{ detect: vi.fn(async () => detection), isInteractive: () => false, ensureServiceShim, success },
);
expect(installed).toBe(true);
expect(ensureServiceShim).toHaveBeenCalledOnce();
expect(success).toHaveBeenCalledWith(expect.stringContaining("managed paperclipai payload"));
expect(detection.manager.install).toHaveBeenCalledWith({ startNow: true, startOnLogin: true });
});
it("declines instead of installing a service without a binary", async () => {
const detection = supportedDetection();
const warn = vi.fn();
const installed = await handleOnboardService(
{ yes: true, installService: true },
{
detect: vi.fn(async () => detection),
isInteractive: () => false,
ensureServiceShim: vi.fn(async () => ({ ok: false, installedNow: false, reason: "npm exploded" })),
warn,
},
);
expect(installed).toBe(false);
expect(detection.manager.install).not.toHaveBeenCalled();
expect(warn).toHaveBeenCalledWith(expect.stringContaining("npm exploded"));
expect(warn).toHaveBeenCalledWith(expect.stringContaining("paperclipai install"));
});
});
describe("isInstallableReleaseVersion", () => {
it("accepts calendar releases and rejects placeholders", () => {
expect(isInstallableReleaseVersion("2026.824.1")).toBe(true);
expect(isInstallableReleaseVersion("2026.818.0-beta.1")).toBe(true);
expect(isInstallableReleaseVersion("0.3.1")).toBe(false);
expect(isInstallableReleaseVersion("not-a-version")).toBe(false);
});
});
describe("onboarded service dashboard handoff", () => {
it("resolves a reachable local dashboard URL", () => {
expect(resolveOnboardServiceDashboardUrl(dashboardConfig({ host: "0.0.0.0", port: 4321 })))
.toBe("http://127.0.0.1:4321");
expect(resolveOnboardServiceDashboardUrl(dashboardConfig({ host: "::1" })))
.toBe("http://[::1]:3100");
});
it("uses the configured public URL when auth requires one", () => {
expect(resolveOnboardServiceDashboardUrl(dashboardConfig({
baseUrlMode: "explicit",
publicBaseUrl: "https://paperclip.example.com/",
}))).toBe("https://paperclip.example.com");
});
it("prints the dashboard URL without opening a browser in non-interactive runs", async () => {
const info = vi.fn();
const waitUntilReady = vi.fn(async () => ({
schemaVersion: 1 as const,
instanceId: "default",
pid: 123,
host: "127.0.0.1",
port: 3100,
dashboardUrl: "http://127.0.0.1:3100",
startedAt: "2026-08-25T00:00:00.000Z",
}));
const openDashboard = vi.fn(async () => true);
await handoffToOnboardedService(dashboardConfig(), {
isInteractive: () => false,
waitUntilReady,
openDashboard,
info,
});
expect(info).toHaveBeenCalledWith(expect.stringContaining("http://127.0.0.1:3100"));
expect(waitUntilReady).toHaveBeenCalledOnce();
expect(openDashboard).not.toHaveBeenCalled();
});
it("uses the ready service runtime port before opening the dashboard", async () => {
const waitUntilReady = vi.fn(async () => ({
schemaVersion: 1 as const,
instanceId: "default",
pid: 123,
host: "127.0.0.1",
port: 3101,
dashboardUrl: "http://127.0.0.1:3101",
startedAt: "2026-08-25T00:00:00.000Z",
}));
const openDashboard = vi.fn(async () => true);
const success = vi.fn();
await handoffToOnboardedService(dashboardConfig(), {
isInteractive: () => true,
waitUntilReady,
openDashboard,
info: vi.fn(),
success,
});
expect(waitUntilReady).toHaveBeenCalledOnce();
expect(openDashboard).toHaveBeenCalledWith("http://127.0.0.1:3101");
expect(success).toHaveBeenCalledWith(expect.stringContaining("Sent"));
});
it("keeps the manual link and warns when service health does not become ready", async () => {
const openDashboard = vi.fn(async () => true);
const warn = vi.fn();
await handoffToOnboardedService(dashboardConfig(), {
isInteractive: () => true,
waitUntilReady: vi.fn(async () => null),
openDashboard,
info: vi.fn(),
warn,
});
expect(openDashboard).not.toHaveBeenCalled();
expect(warn).toHaveBeenCalledWith(expect.stringContaining("paperclipai service logs"));
});
});
describe("shouldOfferForegroundStart", () => {
const base = { serviceInstalled: false, startAlreadyDecided: false, invokedByRun: false, interactive: true };
it("offers a foreground start on a plain interactive onboard", () => {
expect(shouldOfferForegroundStart(base)).toBe(true);
});
it("never prompts after the service was installed and started", () => {
expect(shouldOfferForegroundStart({ ...base, serviceInstalled: true })).toBe(false);
});
it("never prompts when the start decision was already made by flags", () => {
expect(shouldOfferForegroundStart({ ...base, startAlreadyDecided: true })).toBe(false);
});
it("never prompts when run itself invoked onboarding", () => {
expect(shouldOfferForegroundStart({ ...base, invokedByRun: true })).toBe(false);
});
it("never prompts without an interactive terminal", () => {
expect(shouldOfferForegroundStart({ ...base, interactive: false })).toBe(false);
});
});
+56
View File
@@ -0,0 +1,56 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
readRuntimeInfo,
removeRuntimeInfoForPid,
writeRuntimeInfo,
type PaperclipRuntimeInfo,
} from "../runtime-info.js";
const roots: string[] = [];
afterEach(() => {
for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
});
function fixture(): { filePath: string; info: PaperclipRuntimeInfo } {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-runtime-info-"));
roots.push(root);
return {
filePath: path.join(root, "runtime-info.json"),
info: {
schemaVersion: 1,
instanceId: "default",
pid: 123,
host: "127.0.0.1",
port: 3101,
dashboardUrl: "http://127.0.0.1:3101",
startedAt: "2026-08-25T00:00:00.000Z",
},
};
}
describe("runtime info", () => {
it("writes and reads the selected runtime endpoint", () => {
const { filePath, info } = fixture();
writeRuntimeInfo(info, filePath);
expect(readRuntimeInfo("default", filePath)).toEqual(info);
});
it("does not remove runtime info owned by a replacement process", () => {
const { filePath, info } = fixture();
writeRuntimeInfo(info, filePath);
removeRuntimeInfoForPid(999, "default", filePath);
expect(readRuntimeInfo("default", filePath)).toEqual(info);
removeRuntimeInfoForPid(info.pid, "default", filePath);
expect(readRuntimeInfo("default", filePath)).toBeNull();
});
it("rejects malformed runtime info", () => {
const { filePath } = fixture();
fs.writeFileSync(filePath, JSON.stringify({ schemaVersion: 1, port: 70_000 }));
expect(readRuntimeInfo("default", filePath)).toBeNull();
});
});
@@ -3,6 +3,13 @@ import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { serviceHealthChecks } from "../checks/service-health-check.js";
import {
extractExecutableFromLaunchdPlist,
extractExecutableFromSystemdUnit,
isExecutableFile,
renderLaunchdPlist,
renderSystemdUnit,
} from "../services/service-manager.js";
import { resolveRestartExpectedVersion, withHotRestartLock } from "../commands/service.js";
import type { PaperclipConfig } from "../config/schema.js";
import { buildLocalHealthUrl } from "../utils/health-url.js";
@@ -52,6 +59,7 @@ function managerFixture(active = true) {
linger: true,
})),
logs: vi.fn(async () => undefined),
installedExecutablePath: vi.fn(async () => null),
};
}
@@ -129,6 +137,7 @@ describe("service health doctor checks", () => {
const results = await serviceHealthChecks(config, {
detect: vi.fn(async () => ({ supported: true as const, manager })),
probe: vi.fn(async () => ({ ok: true, version: "1.2.3" })),
shimPresent: vi.fn(async () => true),
});
expect(results).toContainEqual(
@@ -140,3 +149,107 @@ describe("service health doctor checks", () => {
);
});
});
describe("isExecutableFile", () => {
it("accepts only executable regular files", async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "shim-check-"));
const executable = path.join(dir, "exec");
const plain = path.join(dir, "plain");
fs.writeFileSync(executable, "#!/bin/sh\n", { mode: 0o755 });
fs.writeFileSync(plain, "data", { mode: 0o644 });
await expect(isExecutableFile(executable)).resolves.toBe(true);
await expect(isExecutableFile(plain)).resolves.toBe(false);
await expect(isExecutableFile(dir)).resolves.toBe(false);
await expect(isExecutableFile(path.join(dir, "missing"))).resolves.toBe(false);
});
});
describe("service runtime shim awareness", () => {
function inactiveManager() {
return {
platform: "launchd" as const,
instanceId: "default",
serviceName: "ing.paperclip.paperclipai",
definitionPath: "/tmp/nonexistent-definition.plist",
renderDefinition: () => "plist",
install: vi.fn(async () => ({ changed: false })),
uninstall: vi.fn(async () => undefined),
start: vi.fn(async () => undefined),
stop: vi.fn(async () => undefined),
restart: vi.fn(async () => undefined),
status: vi.fn(async () => ({
platform: "launchd" as const,
serviceName: "ing.paperclip.paperclipai",
installed: true,
active: false,
enabled: true,
pid: null,
detail: "loaded",
})),
logs: vi.fn(async () => undefined),
installedExecutablePath: vi.fn(async (): Promise<string | null> => null),
};
}
it("blames the missing binary, not a port conflict, when the shim is gone", async () => {
const results = await serviceHealthChecks({} as never, {
detect: vi.fn(async () => ({ supported: true as const, manager: inactiveManager() as never })),
probe: vi.fn(async () => ({ ok: false, version: null, error: "fetch failed" })),
shimPresent: vi.fn(async () => false),
});
const runtime = results.find((r) => r.name === "Service runtime");
expect(runtime?.status).toBe("fail");
expect(runtime?.message).toContain("no executable exists at");
expect(runtime?.repairHint).toContain("paperclipai install");
});
it("diagnoses against the executable recorded in the definition, not the current env", async () => {
const manager = inactiveManager();
manager.installedExecutablePath = vi.fn(async () => "/custom/bin/paperclipai");
const shimPresent = vi.fn(async () => false);
const results = await serviceHealthChecks({} as never, {
detect: vi.fn(async () => ({ supported: true as const, manager: manager as never })),
probe: vi.fn(async () => ({ ok: false, version: null, error: "fetch failed" })),
shimPresent,
});
const runtime = results.find((r) => r.name === "Service runtime");
expect(shimPresent).toHaveBeenCalledWith("/custom/bin/paperclipai");
expect(runtime?.message).toContain("/custom/bin/paperclipai");
expect(runtime?.repairHint).toContain("/custom/bin/paperclipai");
expect(runtime?.repairHint).toContain("unset PAPERCLIP_SHIM_PATH");
expect(runtime?.repairHint).toContain("`paperclipai install` followed by `paperclipai service install`");
});
it("attributes a healthy foreign responder instead of reporting Healthy", async () => {
const results = await serviceHealthChecks({} as never, {
detect: vi.fn(async () => ({ supported: true as const, manager: inactiveManager() as never })),
probe: vi.fn(async () => ({ ok: true, version: "9.9.9" })),
shimPresent: vi.fn(async () => true),
});
const healthResult = results.find((r) => r.name === "Service health");
expect(healthResult?.status).toBe("warn");
expect(healthResult?.message).toContain("but not from ing.paperclip.paperclipai");
const runtime = results.find((r) => r.name === "Service runtime");
expect(runtime?.message).toContain("serving another Paperclip process");
});
});
describe("definition executable extraction", () => {
it("round-trips through both renderers", () => {
const unit = renderSystemdUnit({ instanceId: "default", shimPath: "/custom/bin/paperclipai", homeDir: "/home/x/.paperclip" });
expect(extractExecutableFromSystemdUnit(unit)).toBe("/custom/bin/paperclipai");
const plist = renderLaunchdPlist({ instanceId: "default", shimPath: "/custom/bin/paperclipai", homeDir: "/home/x/.paperclip", stdoutPath: "/tmp/o.log", stderrPath: "/tmp/e.log" });
expect(extractExecutableFromLaunchdPlist(plist)).toBe("/custom/bin/paperclipai");
expect(extractExecutableFromSystemdUnit("garbage")).toBe(null);
expect(extractExecutableFromLaunchdPlist("garbage")).toBe(null);
});
it("round-trips paths the renderers escape", () => {
const hostile = '/tmp/we"ird $pa%th & <x>/paperclipai';
const unit = renderSystemdUnit({ instanceId: "default", shimPath: hostile, homeDir: "/home/x/.paperclip" });
expect(extractExecutableFromSystemdUnit(unit)).toBe(hostile);
const plist = renderLaunchdPlist({ instanceId: "default", shimPath: hostile, homeDir: "/home/x/.paperclip", stdoutPath: "/tmp/o.log", stderrPath: "/tmp/e.log" });
expect(extractExecutableFromLaunchdPlist(plist)).toBe(hostile);
});
});
+7 -1
View File
@@ -1610,7 +1610,13 @@ describe("worktree helpers", () => {
fs.rmSync(tempRoot, { recursive: true, force: true });
}
},
30_000,
// This test starts three separate embedded Postgres lifecycles (the
// source database, the worktree init's internal target database, and a
// third instance opened here to verify the seeded rows), so it needs
// more headroom than the other embedded-Postgres tests in this file.
// It normally finishes in well under 10s; the 60s budget absorbs CI
// runner contention without masking a real hang.
60_000,
);
itEmbeddedPostgres(
+44 -14
View File
@@ -1,9 +1,12 @@
import fs from "node:fs/promises";
import path from "node:path";
import type { PaperclipConfig } from "../config/schema.js";
import { resolvePaperclipInstanceId } from "../config/home.js";
import { readInstallManifest } from "../install-store.js";
import { readInstallManifest, resolveInstallStorePaths } from "../install-store.js";
import {
detectServiceManager,
isExecutableFile,
resolveServiceShimPath,
type ServiceManagerDetection,
} from "../services/service-manager.js";
import { buildLocalHealthUrl } from "../utils/health-url.js";
@@ -13,6 +16,7 @@ type HealthResult = { ok: boolean; version: string | null; error?: string };
type ServiceCheckDependencies = {
detect: (instanceId: string) => Promise<ServiceManagerDetection>;
probe: (config: PaperclipConfig) => Promise<HealthResult>;
shimPresent: (executablePath: string) => Promise<boolean>;
};
async function probeHealth(config: PaperclipConfig): Promise<HealthResult> {
@@ -45,6 +49,7 @@ export async function serviceHealthChecks(
const deps: ServiceCheckDependencies = {
detect: (instanceId) => detectServiceManager({ instanceId }),
probe: probeHealth,
shimPresent: (executablePath) => isExecutableFile(executablePath),
...dependencies,
};
const instanceId = resolvePaperclipInstanceId();
@@ -84,17 +89,36 @@ export async function serviceHealthChecks(
);
const health = await deps.probe(config);
// The installed definition is the truth about what the service executes;
// fall back to the environment-derived path only when it is unreadable.
const serviceExecutable = (await manager.installedExecutablePath()) ?? resolveServiceShimPath();
const shimPresent = status.active ? true : await deps.shimPresent(serviceExecutable);
results.push(
status.active
? { name: "Service runtime", status: "pass", message: `${status.serviceName} is active` }
: {
name: "Service runtime",
status: "fail",
message: health.ok
? `${status.serviceName} is inactive but the configured port is serving another Paperclip process`
: `${status.serviceName} is ${status.detail ?? "inactive"}`,
repairHint: "Run `paperclipai service start`, or stop the conflicting foreground process first",
},
: !shimPresent
? {
name: "Service runtime",
status: "fail",
message: `${status.serviceName} cannot start: no executable exists at ${serviceExecutable}`,
repairHint:
path.resolve(serviceExecutable) === path.resolve(resolveInstallStorePaths().shimPath)
? "Run `paperclipai install` to restore the managed payload and shim, then `paperclipai service start`"
: `Restore the executable at ${serviceExecutable}, or unset PAPERCLIP_SHIM_PATH and run \`paperclipai install\` followed by \`paperclipai service install\` to re-point the service at the managed shim`,
}
: health.ok
? {
name: "Service runtime",
status: "fail",
message: `${status.serviceName} is inactive but the configured port is serving another Paperclip process`,
repairHint: "Run `paperclipai service start`, or stop the conflicting foreground process first",
}
: {
name: "Service runtime",
status: "fail",
message: `${status.serviceName} is ${status.detail ?? "inactive"}`,
repairHint: "Run `paperclipai service start`; inspect `paperclipai service logs` if it does not stay up",
},
);
let expectedVersion: string | null = null;
@@ -116,11 +140,17 @@ export async function serviceHealthChecks(
message: `Running ${health.version ?? "unknown"}; managed install is ${expectedVersion}`,
repairHint: "Run `paperclipai service restart --expected-version " + expectedVersion + "`",
}
: {
name: "Service health",
status: "pass",
message: `Healthy${health.version ? ` at version ${health.version}` : ""}`,
},
: status.active
? {
name: "Service health",
status: "pass",
message: `Healthy${health.version ? ` at version ${health.version}` : ""}`,
}
: {
name: "Service health",
status: "warn",
message: `The configured port answers healthy${health.version ? ` (version ${health.version})` : ""}, but not from ${status.serviceName} — the service is inactive`,
},
);
if (status.enabled && status.linger === false) {
+13 -3
View File
@@ -52,7 +52,11 @@ import {
trackInstallStarted,
trackInstallCompleted,
} from "../telemetry.js";
import { handleOnboardService } from "../onboard-service.js";
import {
handleOnboardService,
handoffToOnboardedService,
shouldOfferForegroundStart,
} from "../onboard-service.js";
import { readInstallManifest, isManagedExecutable } from "../install-store.js";
type SetupMode = "quickstart" | "advanced";
@@ -457,9 +461,12 @@ export async function onboard(opts: OnboardOptions): Promise<void> {
printManagedInstallHint();
const serviceInstalled = await handleOnboardService(opts);
if (serviceInstalled) {
await handoffToOnboardedService(existingConfig);
}
let shouldRunNow = !serviceInstalled && (opts.run === true || opts.yes === true);
if (!shouldRunNow && !opts.invokedByRun && process.stdin.isTTY && process.stdout.isTTY) {
if (shouldOfferForegroundStart({ serviceInstalled, startAlreadyDecided: shouldRunNow, invokedByRun: opts.invokedByRun === true, interactive: Boolean(process.stdin.isTTY && process.stdout.isTTY) })) {
const answer = await p.confirm({
message: "Start Paperclip now?",
initialValue: true,
@@ -723,9 +730,12 @@ export async function onboard(opts: OnboardOptions): Promise<void> {
}
const serviceInstalled = await handleOnboardService(opts);
if (serviceInstalled) {
await handoffToOnboardedService(config);
}
let shouldRunNow = !serviceInstalled && (opts.run === true || opts.yes === true);
if (!shouldRunNow && !opts.invokedByRun && process.stdin.isTTY && process.stdout.isTTY) {
if (shouldOfferForegroundStart({ serviceInstalled, startAlreadyDecided: shouldRunNow, invokedByRun: opts.invokedByRun === true, interactive: Boolean(process.stdin.isTTY && process.stdout.isTTY) })) {
const answer = await p.confirm({
message: "Start Paperclip now?",
initialValue: true,
+11
View File
@@ -17,6 +17,7 @@ import {
resolvePaperclipInstanceId,
} from "../config/home.js";
import { assertForegroundRunAllowed } from "../services/service-manager.js";
import { removeRuntimeInfoForPid, writeRuntimeInfo } from "../runtime-info.js";
import { printUpdateNotice } from "../update-notice.js";
import { ensureWorktreeSeeded } from "./worktree.js";
@@ -93,6 +94,16 @@ export async function runCommand(opts: RunOptions): Promise<void> {
p.log.step("Starting Paperclip server...");
const startedServer = await importServerEntry();
writeRuntimeInfo({
schemaVersion: 1,
instanceId,
pid: process.pid,
host: startedServer.host,
port: startedServer.listenPort,
dashboardUrl: startedServer.apiUrl.replace(/\/api\/?$/, ""),
startedAt: new Date().toISOString(),
});
process.once("exit", () => removeRuntimeInfoForPid(process.pid, instanceId));
if (shouldGenerateBootstrapInviteAfterStart(config)) {
p.log.step("Generating bootstrap CEO invite");
+199
View File
@@ -1,18 +1,130 @@
import path from "node:path";
import * as p from "@clack/prompts";
import pc from "picocolors";
import type { PaperclipConfig } from "./config/schema.js";
import { openUrl } from "./client/board-auth.js";
import { installCommand } from "./commands/install.js";
import { resolvePaperclipInstanceId } from "./config/home.js";
import { readRuntimeInfo, type PaperclipRuntimeInfo } from "./runtime-info.js";
import {
readInstallManifest,
resolveInstallStorePaths,
type InstallManifest,
} from "./install-store.js";
import {
detectServiceManager,
isExecutableFile,
resolveServiceShimPath,
type ServiceManagerDetection,
} from "./services/service-manager.js";
import { buildLocalAppUrl, buildLocalHealthUrl } from "./utils/health-url.js";
import { packageVersion } from "./version.js";
export type OnboardServiceOptions = {
yes?: boolean;
installService?: boolean;
};
type EnsureShimResult = { ok: boolean; installedNow: boolean; reason?: string };
type OnboardServiceDashboardConfig = {
auth: Pick<PaperclipConfig["auth"], "baseUrlMode" | "publicBaseUrl">;
server: Pick<PaperclipConfig["server"], "host" | "port">;
};
type OnboardServiceDashboardDependencies = {
isInteractive: () => boolean;
waitUntilReady: () => Promise<PaperclipRuntimeInfo | null>;
openDashboard: (url: string) => Promise<boolean>;
info: (message: string) => void;
success: (message: string) => void;
warn: (message: string) => void;
};
function envDisablesBrowser(value = process.env.PAPERCLIP_NO_BROWSER): boolean {
const normalized = value?.trim().toLowerCase();
return normalized === "1" || normalized === "true" || normalized === "yes";
}
async function waitUntilDashboardReady(timeoutMs = 60_000): Promise<PaperclipRuntimeInfo | null> {
const instanceId = resolvePaperclipInstanceId();
const detection = await detectServiceManager({ instanceId });
if (!detection.supported) return null;
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const info = readRuntimeInfo(instanceId);
if (info) {
const status = await detection.manager.status().catch(() => null);
if (status?.active && status.pid === info.pid) {
try {
const response = await fetch(buildLocalHealthUrl(info.host, info.port), {
signal: AbortSignal.timeout(2_000),
});
const body = await response.json() as { status?: unknown };
if (response.ok && body.status === "ok") return info;
} catch {}
}
}
await new Promise((resolve) => setTimeout(resolve, 500));
}
return null;
}
const defaultDashboardDependencies: OnboardServiceDashboardDependencies = {
isInteractive: () => process.stdin.isTTY === true && process.stdout.isTTY === true,
waitUntilReady: waitUntilDashboardReady,
openDashboard: openUrl,
info: (message) => p.log.info(message),
success: (message) => p.log.success(message),
warn: (message) => p.log.warn(message),
};
export function resolveOnboardServiceDashboardUrl(
config: OnboardServiceDashboardConfig,
runtime?: Pick<PaperclipRuntimeInfo, "dashboardUrl"> | null,
): string {
if (runtime?.dashboardUrl.trim()) return runtime.dashboardUrl.trim().replace(/\/+$/, "");
if (config.auth.baseUrlMode === "explicit" && config.auth.publicBaseUrl?.trim()) {
return config.auth.publicBaseUrl.trim().replace(/\/+$/, "");
}
return buildLocalAppUrl(config.server.host, config.server.port);
}
export async function handoffToOnboardedService(
config: OnboardServiceDashboardConfig,
dependencies: Partial<OnboardServiceDashboardDependencies> = {},
): Promise<void> {
const deps = { ...defaultDashboardDependencies, ...dependencies };
const runtime = await deps.waitUntilReady();
const dashboardUrl = resolveOnboardServiceDashboardUrl(config, runtime);
deps.info(`Paperclip dashboard: ${pc.cyan(dashboardUrl)}`);
if (!runtime) {
deps.warn(
`The background service started, but the dashboard is not ready yet. ` +
`Open ${dashboardUrl} after checking \`paperclipai service logs\`.`,
);
return;
}
if (!deps.isInteractive() || envDisablesBrowser()) return;
if (await deps.openDashboard(dashboardUrl)) {
deps.success("Sent the Paperclip dashboard to your browser.");
} else {
deps.warn(`Could not open a browser automatically. Open ${dashboardUrl} manually.`);
}
}
// Source checkouts carry the repository placeholder version; installing
// that as an npm spec would fetch an ancient release (or nothing) instead
// of the running code. Only real calendar versions are installable.
export function isInstallableReleaseVersion(version: string): boolean {
return /^\d{4}\.\d{1,4}\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version);
}
type OnboardServiceDependencies = {
detect: (instanceId: string) => Promise<ServiceManagerDetection>;
ensureServiceShim: () => Promise<EnsureShimResult>;
confirm: () => Promise<boolean>;
confirmLinger: () => Promise<boolean>;
isInteractive: () => boolean;
@@ -23,6 +135,60 @@ type OnboardServiceDependencies = {
const defaultDependencies: OnboardServiceDependencies = {
detect: (instanceId) => detectServiceManager({ instanceId }),
// The service definition targets the managed shim. An ephemeral run (npx)
// never lays it down, so installing the service without this step creates
// a definition that crash-loops on a missing binary.
ensureServiceShim: async () => {
const shimPath = resolveServiceShimPath();
if (await isExecutableFile(shimPath)) {
return { ok: true, installedNow: false };
}
const storeShimPath = resolveInstallStorePaths().shimPath;
if (path.resolve(shimPath) !== path.resolve(storeShimPath)) {
return {
ok: false,
installedNow: false,
reason: `no executable exists at ${shimPath} (PAPERCLIP_SHIM_PATH), and it is outside the managed install store`,
};
}
let manifest: InstallManifest | null = null;
try {
manifest = readInstallManifest();
} catch {}
try {
if (manifest?.source === "git" && manifest.repo) {
// A managed git payload must be preserved as-is: reinstall the
// exact revision the manifest records, not an npm release.
await installCommand({ repo: manifest.repo, ref: manifest.sha ?? manifest.ref, yes: true });
} else if (isInstallableReleaseVersion(packageVersion)) {
// packageVersion, not cliVersion: a managed executable's cliVersion
// carries provenance text that is not an installable npm spec.
await installCommand({ version: packageVersion, yes: true });
} else {
return {
ok: false,
installedNow: false,
reason:
`this build reports version ${packageVersion}, which is not an installable release; ` +
"run `paperclipai install` (or `paperclipai install --repo <repo> --ref <ref>` for source builds) first",
};
}
} catch (error) {
return {
ok: false,
installedNow: false,
reason: error instanceof Error ? error.message : String(error),
};
}
if (await isExecutableFile(shimPath)) {
return { ok: true, installedNow: true };
}
return {
ok: false,
installedNow: false,
reason: `the managed install completed but no executable shim appeared at ${shimPath}`,
};
},
confirm: async () => {
const answer = await p.confirm({
message: "Install Paperclip as a background service?",
@@ -68,6 +234,22 @@ export async function handleOnboardService(
if (!explicitlyRequested && !(await deps.confirm())) return false;
// A definition pointing at a missing binary crash-loops in the platform
// supervisor's penalty box while doctor blames a port conflict.
// Materialize the managed install first, or decline with the repair path
// instead of installing a corpse.
const shim = await deps.ensureServiceShim();
if (!shim.ok) {
deps.warn(
`Background service not installed: ${shim.reason ?? "the managed install could not be completed"}. ` +
"Run `paperclipai install`, then `paperclipai service install`.",
);
return false;
}
if (shim.installedNow) {
deps.success("Installed the managed paperclipai payload and command shim for the service.");
}
await detection.manager.install({ startNow: true, startOnLogin: true });
if (!explicitlyRequested && detection.manager.enableLinger && await deps.confirmLinger()) {
await detection.manager.enableLinger();
@@ -75,3 +257,20 @@ export async function handleOnboardService(
deps.success(`Installed and started ${detection.manager.serviceName}.`);
return true;
}
// Onboarding falls back to offering a foreground start when nothing else
// will serve. A just-installed service is already serving, so offering the
// start would only run the user into the already-running instance guard.
export function shouldOfferForegroundStart(options: {
serviceInstalled: boolean;
startAlreadyDecided: boolean;
invokedByRun: boolean;
interactive: boolean;
}): boolean {
return (
!options.startAlreadyDecided &&
!options.serviceInstalled &&
!options.invokedByRun &&
options.interactive
);
}
+82
View File
@@ -0,0 +1,82 @@
import fs from "node:fs";
import path from "node:path";
import { resolvePaperclipInstanceRoot } from "./config/home.js";
export const PAPERCLIP_RUNTIME_INFO_FILENAME = "runtime-info.json";
export type PaperclipRuntimeInfo = {
schemaVersion: 1;
instanceId: string;
pid: number;
host: string;
port: number;
dashboardUrl: string;
startedAt: string;
};
export function resolveRuntimeInfoPath(instanceId?: string): string {
return path.join(resolvePaperclipInstanceRoot(instanceId), PAPERCLIP_RUNTIME_INFO_FILENAME);
}
function parseRuntimeInfo(value: unknown): PaperclipRuntimeInfo | null {
if (!value || typeof value !== "object") return null;
const record = value as Record<string, unknown>;
if (
record.schemaVersion !== 1 ||
typeof record.instanceId !== "string" ||
!Number.isInteger(record.pid) ||
(record.pid as number) <= 0 ||
typeof record.host !== "string" ||
!Number.isInteger(record.port) ||
(record.port as number) <= 0 ||
(record.port as number) > 65_535 ||
typeof record.dashboardUrl !== "string" ||
typeof record.startedAt !== "string"
) {
return null;
}
return record as PaperclipRuntimeInfo;
}
export function readRuntimeInfo(instanceId?: string, filePath = resolveRuntimeInfoPath(instanceId)): PaperclipRuntimeInfo | null {
try {
const info = parseRuntimeInfo(JSON.parse(fs.readFileSync(filePath, "utf8")));
if (!info) return null;
if (instanceId && info.instanceId !== instanceId) return null;
return info;
} catch {
return null;
}
}
export function writeRuntimeInfo(
info: PaperclipRuntimeInfo,
filePath = resolveRuntimeInfoPath(info.instanceId),
): void {
const directoryPath = path.dirname(filePath);
fs.mkdirSync(directoryPath, { recursive: true, mode: 0o700 });
const temporaryPath = path.join(
directoryPath,
`.${path.basename(filePath)}.tmp-${process.pid}-${Date.now()}`,
);
try {
fs.writeFileSync(temporaryPath, `${JSON.stringify(info, null, 2)}\n`, {
encoding: "utf8",
mode: 0o600,
flag: "wx",
});
fs.renameSync(temporaryPath, filePath);
} finally {
fs.rmSync(temporaryPath, { force: true });
}
}
export function removeRuntimeInfoForPid(
pid: number,
instanceId?: string,
filePath = resolveRuntimeInfoPath(instanceId),
): void {
const current = readRuntimeInfo(instanceId, filePath);
if (current?.pid !== pid) return;
fs.rmSync(filePath, { force: true });
}
+57
View File
@@ -1,4 +1,5 @@
import fs from "node:fs/promises";
import { constants as fsConstants } from "node:fs";
import os from "node:os";
import path from "node:path";
import { execFile } from "node:child_process";
@@ -33,6 +34,7 @@ export interface ServiceManager {
restart(): Promise<void>;
status(): Promise<ServiceStatus>;
logs(follow: boolean, lines: number): Promise<void>;
installedExecutablePath(): Promise<string | null>;
enableLinger?(): Promise<void>;
}
@@ -75,6 +77,45 @@ export function resolveServiceShimPath(homeDir = os.homedir()): string {
return process.env.PAPERCLIP_SHIM_PATH?.trim() || path.join(homeDir, ".local", "bin", "paperclipai");
}
// The installed definition, not the current environment, is the truth
// about what the service executes: PAPERCLIP_SHIM_PATH may have changed
// or been unset since the definition was written.
function unescapeSystemd(value: string): string {
return value.replace(/\\\\|\\"|\$\$|%%/g, (m) =>
m === "\\\\" ? "\\" : m === '\\"' ? '"' : m === "$$" ? "$" : "%",
);
}
function unescapeXml(value: string): string {
return value.replace(/&(amp|lt|gt|quot|apos);/g, (_, name: string) =>
name === "amp" ? "&" : name === "lt" ? "<" : name === "gt" ? ">" : name === "quot" ? '"' : "'",
);
}
export function extractExecutableFromSystemdUnit(content: string): string | null {
const match = content.match(/^ExecStart="((?:\\.|[^"\\])*)"/m);
return match ? unescapeSystemd(match[1]) : null;
}
export function extractExecutableFromLaunchdPlist(content: string): string | null {
const match = content.match(/<key>ProgramArguments<\/key>\s*<array>\s*<string>([^<]+)<\/string>/);
return match ? unescapeXml(match[1]) : null;
}
// The service definition executes this path directly: existence is not
// enough — a directory or a non-executable file would satisfy fs.access's
// default mode and still crash the supervisor at spawn.
export async function isExecutableFile(filePath: string): Promise<boolean> {
try {
const stats = await fs.stat(filePath);
if (!stats.isFile()) return false;
await fs.access(filePath, fsConstants.X_OK);
return true;
} catch {
return false;
}
}
export function systemdServiceName(instanceId: string): string {
return instanceId === "default" ? "paperclipai.service" : `paperclipai-${instanceId}.service`;
}
@@ -174,6 +215,14 @@ export class SystemdServiceManager implements ServiceManager {
return renderSystemdUnit({ instanceId: this.instanceId, shimPath: this.shimPath, homeDir: this.homeDir });
}
async installedExecutablePath(): Promise<string | null> {
try {
return extractExecutableFromSystemdUnit(await fs.readFile(this.definitionPath, "utf8"));
} catch {
return null;
}
}
private async ensureCurrent(): Promise<boolean> {
const changed = await writeIfChanged(this.definitionPath, this.renderDefinition());
if (changed) await this.runner("systemctl", ["--user", "daemon-reload"]);
@@ -242,6 +291,14 @@ export class LaunchdServiceManager implements ServiceManager {
renderDefinition(): string { return renderLaunchdPlist({ instanceId: this.instanceId, shimPath: this.shimPath, homeDir: this.homeDir, stdoutPath: this.stdoutPath, stderrPath: this.stderrPath }); }
async installedExecutablePath(): Promise<string | null> {
try {
return extractExecutableFromLaunchdPlist(await fs.readFile(this.definitionPath, "utf8"));
} catch {
return null;
}
}
async install(options: ServiceInstallOptions): Promise<{ changed: boolean }> {
await fs.mkdir(path.dirname(this.stdoutPath), { recursive: true });
const changed = await writeIfChanged(this.definitionPath, this.renderDefinition());
+6 -2
View File
@@ -1,4 +1,4 @@
export function buildLocalHealthUrl(host: string | undefined, port: number): string {
export function buildLocalAppUrl(host: string | undefined, port: number): string {
const configuredHost = host?.trim();
const reachableHost = !configuredHost || configuredHost === "0.0.0.0" || configuredHost === "::"
? "127.0.0.1"
@@ -6,5 +6,9 @@ export function buildLocalHealthUrl(host: string | undefined, port: number): str
const urlHost = reachableHost.includes(":") && !reachableHost.startsWith("[")
? `[${reachableHost}]`
: reachableHost;
return `http://${urlHost}:${port}/api/health`;
return `http://${urlHost}:${port}`;
}
export function buildLocalHealthUrl(host: string | undefined, port: number): string {
return `${buildLocalAppUrl(host, port)}/api/health`;
}
+33
View File
@@ -198,6 +198,39 @@ Triage writes serialize on the company and attention-source identity so concurre
`decision_retention` tracks the last observed source `activityAt`, Keep, reversible archive provenance, and monotonic source/archive versions. `decision_archive_notification_outbox` has a unique key over company, source identity, archive version, and immutable origin agent so repeated sweeps cannot enqueue duplicate notifications; delivery claims are retryable and coalesced per agent.
## Native runner persistence
Native runner state is additive to the existing heartbeat tables. Every existing
`heartbeat_runs` row defaults to `runtime_mode = 'legacy'`; adding these columns
does not select the native runtime or start a runner process. Native execution can
record its resolved runtime profile, provider session, driver, completion
contract, durable event cursor, and finalization phase on the run when a later
rollout explicitly selects it.
`completion_contracts`, `native_run_results`, `native_run_finalizations`,
`work_assessments`, `status_decisions`, and `status_decision_effects` form the
append-oriented evidence and status-decision chain. Unique fingerprints,
versions, ordinals, and idempotency keys make retries deterministic. Composite
foreign keys bind every contract, result, assessment, decision, effect, and
finalization to one company, issue, and run. The database rejects mixed-owner
evidence even when every referenced ID exists. Native source identities on
`heartbeat_run_events` are nullable so legacy events remain readable without
rewriting historical rows. Per-run native source identifiers are unique, while
the existing legacy sequence behavior remains unchanged. The hidden native
coordinator serializes on its bound `heartbeat_runs` row, allocates
`next_event_seq`, and commits a validated PRP event before the transport sends
its cumulative ACK. Byte-equivalent source retries return the existing cursor;
gaps and conflicting replays fail closed. Accepted structured results enter the
finalization ledger, whose retry time and owner lease are checked under a row
lock. None of these writes selects a runtime or changes a legacy run's execution
path.
Issue `status_version` advances only when `status` changes. The JavaScript backup
path includes user-defined functions and triggers so a restored database keeps
that invariant. Removing or disabling a future native rollout flag must not
delete these records; persisted experimental runs remain available for recovery
and inspection.
## Plugin database namespaces
The plugin runtime tracks plugin-owned database namespaces and migrations in `plugin_database_namespaces` and `plugin_migrations`. Hosted deployments that separate runtime and migration connections should set `DATABASE_MIGRATION_URL`; plugin namespace migration work uses the migration connection when present.
+5
View File
@@ -138,6 +138,11 @@ paperclipai onboard --yes --install-service # explicit automation opt-in
paperclipai onboard --yes --no-install-service
```
After onboarding installs and starts the service, it waits for the service to
report its selected runtime port and then prints the dashboard URL. Interactive
terminals open that URL in the default browser; headless and non-interactive
runs print the URL without trying to launch a browser.
Service commands are namespaced:
```sh
+8 -1
View File
@@ -558,7 +558,7 @@ Detailed ownership, execution, blocker, active-run watchdog, crash-recovery, and
|---|---|---|
| Create company | yes | no |
| Hire/create agent | yes (direct) | request via approval |
| Pause/resume agent | yes | no |
| Pause/resume agent | yes | pause: no; resume: direct `agents:configure` grant only |
| Create/update task | yes | yes |
| Force reassign task | yes | limited |
| Approve strategy/hire requests | yes | no |
@@ -569,6 +569,13 @@ Detailed ownership, execution, blocker, active-run watchdog, crash-recovery, and
| Manage another user's inbox state | yes | saved target-user opt-in or scoped `inbox:manage` grant |
| Set work-object visibility (issue/project) | no | no (pro gate) |
Agent resume is the only grant-gated exception in the lifecycle-route group. An
agent actor calling `POST /agents/:agentId/resume` must pass the protected
`agent_config:update` decision with `scope.requiresChangeGrant: true`; self
access does not bypass that decision, and `agents:suggest-changes` alone cannot
apply the lifecycle change. Pause, clear-error, terminate, approval, and
key-management routes remain board-only.
### 9.3.1 Shared default-open issue writes
For standard-trust agents, issue comments, issue field/status updates, child
+9 -7
View File
@@ -80,13 +80,15 @@ second run blocked on the lease until the first run fully returns, so the second
run never re-stages into a workspace the first run still uses. The release runs in
a `finally`, so an earlier teardown fault never strands the lease.
## Per-phase telemetry
## Per-phase run-log events
The run emits one telemetry event per named lifecycle phase. Each event carries
only the phase name, the wall-time duration, and the outcome (`ok` or `failed`).
The phase name is one member of a closed allowlist. An event never carries a
The run writes one [run-log event](run-log-events.md) per named lifecycle
phase, to the `heartbeat_run_events` table. This event is not a Paperclip
Telemetry event and not an OpenTelemetry export. Each event carries only the
phase name, the wall-time duration, and the outcome (`ok` or `failed`). The
phase name is one member of a closed allowlist. An event never carries a
command, an argument, a path, an environment value, or a raw identifier. A
telemetry failure never fails the run.
run-log write failure never fails the run.
## Known limitations and deferred work
@@ -97,5 +99,5 @@ telemetry failure never fails the run.
run-scoped credential rebind protocol exists.
- **The sandbox staged-files reuse stays enabled.** Its reuse payload carries no
credential, so a compatible resume reuses the already-staged runtime.
- **The per-phase telemetry is observability-only.** It records the duration and
the outcome of each phase; it does not change run control flow.
- **The per-phase run-log events record the duration and the outcome only.**
They never change run control flow.
@@ -0,0 +1,206 @@
# Paperclip Runner compatibility and rollout
- Status: Proposed
- Date: 2026-08-24
- Parent decision: [Paperclip Runner architecture](paperclip-runner.md)
## Purpose
This document defines compatibility rules for introducing the experimental
Paperclip Runner. These rules are acceptance criteria for each implementation
change. They are not a migration plan for existing adapters.
## Compatibility invariants
1. Existing adapter selection is authoritative. A direct adapter stays direct.
2. A non-runner run must not start runnerd or open PRP.
3. A non-runner run must not create native result, finalization, status-decision,
or runner-transport records.
4. A non-runner run must not invoke native status arbitration.
5. Direct adapters keep their current transcript, interaction, cancellation,
result, and finalization behavior.
6. Runner-only UI controls depend on persisted runtime facts and are absent from
direct runs.
7. The rollout flag controls fresh runner selection and fresh runner starts.
8. A flag change does not rewrite an agent profile or a persisted run choice.
9. Persisted native data remains readable after the flag is disabled.
10. Recovery may finish an already persisted native run while fresh native
starts remain blocked.
## Runtime selection
The server resolves and persists the runtime once, before provider launch.
| Persisted runtime | Adapter | Flag | Result |
| --- | --- | --- | --- |
| none | Any direct adapter | off or on | Use the existing direct path. |
| none | `paperclip_runner` with Codex | off | Reject the fresh start with a stable rollout-disabled error. |
| none | `paperclip_runner` with Codex | on | Use PRP v1 and runnerd. |
| none | `paperclip_runner` with another provider | on | Reject the unsupported provider before runnerd starts. |
| direct | Any | changed later | Keep the persisted direct path. |
| native | Any | changed later | Keep the persisted native path for read, cancel, recovery, and finalization. |
The server must not fall back from a selected `paperclip_runner` start to
`codex_local`. A configuration or rollout error must be visible. Silent fallback
would hide the runtime that executed the task.
## Direct adapter boundary
This rule applies to every built-in and plugin direct adapter. It includes:
- `codex_local`;
- `claude_local`;
- `opencode_local`;
- other local CLI or session adapters;
- process and HTTP adapters;
- gateway adapters; and
- external adapter plugins.
Adding Paperclip Runner must not add runner imports or runner branches inside a
direct adapter implementation. The heartbeat coordinator may select the
explicit runner adapter at one narrow seam. All other adapters continue through
their existing code.
For a flag-off `codex_local` heartbeat, compatibility proof must show:
- one direct invocation;
- the same normalized result and finalization bytes as the approved baseline;
- zero runner processes;
- zero PRP connections; and
- zero native rows.
## Configuration behavior
When the rollout flag is off:
- creation UI does not offer `paperclip_runner`;
- edit UI does not offer switching to `paperclip_runner`;
- server creation and import reject a new `paperclip_runner` selection;
- the server rejects a fresh start for an existing runner-configured agent;
- read and export preserve an existing runner configuration;
- unrelated edits to an existing runner-configured agent do not erase its
configuration; and
- switching that agent to a direct adapter remains allowed.
When the rollout flag is on:
- creation, import, and edit accept `paperclip_runner` only with provider
`codex` and valid Codex configuration;
- switching from a direct adapter affects only future unresolved runs; and
- switching away from the runner affects only future unresolved runs.
Server validation is the authority in both states. Import files and API clients
cannot bypass the flag or provider allowlist.
## Persisted native runs
The following data remains readable independent of the current flag:
- the persisted runtime selection and reason;
- run, runner, session, turn, and provider identity;
- ordered runner and provider events;
- accepted result and evidence assessment;
- finalization coordinator and status decision;
- usage, cost, cancellation, and terminal details; and
- durable interactions and final task-thread reply.
If the flag is disabled during an in-flight native run, the server may reconnect,
cancel, reconcile, and finalize that same run. It must not use that recovery as
authority to start a new native run.
Recovery must remain idempotent. Repeating it cannot add a second final reply,
interaction, wake, status decision, or application effect.
## Task-page compatibility
The task page uses one provider-neutral thread projection. Runtime facts may add
runner event groups, semantic receipts, usage, and structured questions. They
must not replace classic direct-adapter content.
Direct-adapter coverage must include:
- an active run;
- a settled run;
- an empty transcript;
- a pending interaction;
- a resolved interaction; and
- the classic interface state.
The existing composer remains usable for direct adapters. A direct run does not
show reconnect, runner cancel, semantic receipt, or other runner-only controls.
Final replies continue to use the existing issue-comment behavior.
## Structured input compatibility
New structured questions use `paperclip.question_set.v1` and the matching
response contract. Provider-specific question objects remain inside their
drivers.
Legacy unstructured interaction records remain readable and resolvable. A
structured form fails closed when its required schema, question mode, question
ID, option ID, or response value is invalid. The implementation must not silently
convert malformed structured input to a legacy text prompt.
## Protocol version compatibility
PRP wire versions, fixture versions, event schema versions, and typed schema
discriminators are independently versioned.
- Peers negotiate the highest common PRP version.
- No common required version fails closed before command or provider execution.
- An unknown required fixture or event schema version fails closed.
- An unknown required typed discriminator fails closed.
- Additive optional properties remain compatible only when an old v1 consumer
can ignore them without changing behavior.
- Unknown object properties must survive validation when the owning schema
permits additive fields.
Breaking meaning requires a new required version. A provider error is not a
protocol negotiation result.
## Semantic action compatibility
Catalog generation and production authorization are separate steps.
- A catalog entry does not authorize production use.
- An operation without a production binding is undiscoverable.
- An operation denied to the run is undiscoverable unless the protocol
explicitly defines a safe denied receipt for that discovery mode.
- A duplicate call with the same idempotency key and canonical input returns the
original safe receipt.
- Reusing the key with different input returns a conflict and performs no second
effect.
- Receipts redact credentials, private provider payloads, and hidden identity.
## Required compatibility matrix
Each runner-related pull request updates only rows that it can execute. The
complete first-wave matrix must cover:
| Area | Required cases |
| --- | --- |
| Runtime selection | Every built-in direct adapter, explicit runner selection, unsupported provider, flag on, and flag off. |
| Direct regression | Flag-off `codex_local` invocation count, byte-stable result/finalization, and zero native rows. |
| Configuration | Enabled and disabled create, import, edit, read, export, and adapter switch. |
| Recovery | Persisted native run after flag disable, reconnect, duplicate event, duplicate command, cancellation, and server restart. |
| Protocol security | Cross-company binding, ticket replay, ticket expiry, malformed frame, unsupported version, and revoked lease. |
| Semantic actions | Discovery denial, unbound action, duplicate call, conflicting retry, redaction, and governed action. |
| Task page | Active, settled, empty transcript, interaction, and classic direct-adapter states. |
| Structured input | Valid response, malformed response, stale response, duplicate response, provider loss, and cancellation. |
## Pull request acceptance
Every implementation pull request must:
- build and work against its declared base;
- keep its changed-file count below 100;
- update manifests and exports only for implemented surfaces;
- run the smallest relevant tests before the repository handoff gate;
- run typecheck, tests, and build for handoff;
- run protocol parity, Cargo, clean-consumer, migration, token, and browser gates
when those areas change;
- contain no unexplained failed, cancelled, or path-skipped verification; and
- resolve actionable review and security findings before it is ready.
Generated files land with their source and a drift check. `pnpm-lock.yaml` is
owned by CI and is not part of these pull requests.
+286
View File
@@ -0,0 +1,286 @@
# ADR: Paperclip Runner architecture
- Status: Proposed
- Date: 2026-08-24
- Owners: Paperclip control plane and runner maintainers
- Related: [Paperclip Runner compatibility and rollout](paperclip-runner-compatibility.md)
## Context
Paperclip is a control plane. It owns companies, agents, issues, budgets,
approvals, and durable workflow state. Agent providers remain execution
services. Existing adapters invoke those services directly from the Paperclip
server.
Paperclip Runner introduces a separate execution process for provider sessions.
This process needs durable delivery, restart recovery, and governed access to
Paperclip actions. It must not become a second control plane. It must also land
without changing the behavior of existing adapters.
The initial implementation is intentionally narrow. It supports Codex through
an explicit, experimental adapter. Other providers and developer tools remain
outside this decision.
## Decision
Add a standalone package named `@paperclipai/paperclip-runner`. The package owns
the language-neutral Paperclip Runner Protocol (PRP), the Rust runner process,
provider drivers, deterministic replay, and semantic action dispatch contracts.
Add one explicit adapter named `paperclip_runner`. The adapter is available only
when an instance-level, default-off rollout flag is enabled. Its first supported
provider is Codex.
Do not route existing adapters through Paperclip Runner. A direct adapter keeps
its current invocation, transcript, interaction, cancellation, and finalization
paths.
## Goals
- Keep runner process ownership outside the Paperclip server process.
- Preserve Paperclip as the authority for identity, policy, and workflow state.
- Recover a run after runner or network interruption without duplicate effects.
- Expose only actions that the current run is allowed to use.
- Make protocol behavior deterministic across TypeScript and Rust.
- Keep existing adapter behavior unchanged while the runner is experimental.
## Non-goals
- Replace existing direct adapters.
- Move business authorization or issue status policy into Rust.
- Give runnerd a broad Paperclip API credential.
- Support OpenCode, ACPX, Claude Managed, AWS AgentCore, or remote sandboxes in
the first production slice.
- Expose browser SDK, React SDK, eval, lab, or scenario-explorer package entry
points in the initial release.
- Commit recorded screenshots, stress logs, or construction history as product
architecture.
## Topology
The initial local topology is:
```text
Paperclip server
| authenticated PRP v1 WebSocket
v
paperclip-runnerd
| Codex app-server protocol
v
Codex
```
The server opens a native run and launches a verified runnerd artifact in the
realized execution environment. Runnerd opens the outbound PRP connection. It
then owns the provider process group and the durable transport state for that
run.
The browser does not connect to runnerd. It reads projections from the existing
Paperclip APIs and task-thread models.
## Dependency direction
The runner package must build and test without importing Paperclip server, UI,
CLI, database, or other private workspace implementation modules.
```text
JSON Schema and fixtures
|
+------------------+
v v
TypeScript contracts Rust runner core
| |
+--------+---------+
v
deterministic parity
Paperclip server ----implements----> runner public ports
```
The dependency points from an implementation to a contract. The Paperclip
server may implement a public runner port. The runner package must not import
the server implementation.
The initial public package surfaces are:
- `@paperclipai/paperclip-runner` for runtime contracts and clients.
- `@paperclipai/paperclip-runner/testing` for deterministic fakes and
conformance helpers.
Every export must have an implementation and a clean-consumer test before it is
published. Later SDK, eval, and lab surfaces require separate decisions.
## Protocol boundary
PRP v1 uses a WebSocket at:
```text
/api/runner/v1/connect/:runId
```
JSON Schema is the language-neutral protocol authority. TypeScript and Rust use
the same canonical fixtures and must produce the same replay result.
Protocol rules are fail closed:
- Unknown required protocol or schema versions are rejected.
- Unknown required discriminators and enum values are rejected.
- Additive optional object fields may be accepted when v1 consumers can ignore
them safely.
- Frames, headers, durable state, diagnostic tails, and replay windows are
bounded.
- Event identity and ordering remain stable across reconnect and restart.
Provider-native messages do not cross this boundary. Drivers translate them to
provider-neutral PRP events, results, usage, cancellations, and structured
input.
## Trust boundary
The Paperclip server is authoritative for:
- company, agent, issue, run, session, and user attribution;
- rollout and runtime selection;
- action discovery and authorization;
- approval, budget, secret, revision, and workspace policy;
- durable application records and activity history;
- result acceptance, finalization, and issue status.
Runnerd is authoritative only for its local responsibilities:
- provider process supervision;
- provider session and turn transport;
- durable PRP outbox and command receipts;
- stable runner-side event identity;
- bounded process diagnostics.
Runnerd receives a short-lived, one-use bootstrap ticket. The ticket is bound to
the company, agent, issue, run, runner, session, turn, and verified artifact. The
server exchanges it for a short-lived connection lease. Raw tickets are never
stored. Runnerd never receives a broad Paperclip API key.
The server rejects expired, replayed, revoked, cross-company, mismatched,
malformed, oversized, or protocol-incompatible connections. Cancellation,
timeout, supersession, and environment-lease loss revoke runner authority.
## Durable delivery and recovery
Runnerd persists an event before it sends the event. The server acknowledges a
cumulative source cursor. Runnerd removes acknowledged data only after it
validates that cursor.
A reconnect reports the last processed command, the next source sequence, the
last acknowledged source sequence, and the unacknowledged range. The server may
then replay one pending command and accept byte-equivalent event retries.
Command identity is idempotent. Reusing a command ID with the same canonical
input returns the stored result. Reusing the ID with different input fails
closed. A repeated semantic tool call cannot repeat an application effect.
Recovery is bounded. Exhausted storage, reconnect, command, or time limits end
the run with a classified failure instead of an unbounded loop.
## Semantic actions
PRP carries provider-neutral semantic operation IDs. The package may define an
operation catalog, but catalog presence does not grant authority.
For each run, the server projects only operations that have a production
binding and that the current actor may discover. An unbound or unauthorized
operation is absent from discovery. The server validates inputs, executes the
existing application authority, and returns a redacted receipt.
This keeps these invariants in one place:
- company scoping;
- actor and run attribution;
- authorization and approval;
- revision and idempotency checks;
- budget and secret policy;
- activity logging and safe error details.
Runnerd cannot forge identity through tool input. Identity and scope come from
the authenticated connection binding.
## Structured input
Questions use `paperclip.question_set.v1`. Responses use the matching canonical
response contract. Drivers translate between these provider-neutral records and
provider-native input APIs.
The server validates a response at the untrusted API edge and again against the
persisted question set before delivery. A process loss may materialize a durable
task-thread interaction, but cancellation and an already resolved request must
not create a second interaction or continuation.
## Persistence and finalization
Native run records are additive. Existing heartbeat and issue records remain
readable. A runner result is an untrusted claim. It does not directly update an
issue status.
The server validates the result, classifies durable evidence, applies status
policy, commits workflow effects, and records finalization. Each phase is
idempotent and recoverable. Existing direct adapters keep their existing
finalization paths and do not invoke native status arbitration.
Disabling the rollout flag blocks fresh runner starts. It does not make an
already persisted native run unreadable or prevent bounded recovery and
finalization of that run.
## Rollout
The rollout has three gates:
1. The instance flag is enabled.
2. The agent explicitly selects `paperclip_runner`.
3. The adapter selects a supported provider. The initial provider is `codex`.
The adapter is hidden from creation and selection surfaces while the flag is
off. Server validation also rejects a fresh runner selection or start while the
flag is off. UI hiding is not the security boundary.
Runtime selection is persisted before launch. Later setting changes cannot
silently move an in-flight run between the direct and native execution paths.
The detailed compatibility rules are in
[Paperclip Runner compatibility and rollout](paperclip-runner-compatibility.md).
## Observability
The server records structured runner, provider, semantic action, interaction,
usage, and terminal events. Logs and receipts must not contain bootstrap
tickets, leases, provider credentials, secret values, complete environments, or
private host paths.
The task page may project these records through existing thread components. A
runner-only control must depend on persisted runtime facts. It must not depend
only on the agent adapter profile.
## Consequences
This design adds process, protocol, and recovery complexity. In return, it gives
provider sessions a durable and testable execution boundary without moving
Paperclip governance into the runner.
The default-off, explicit-adapter rollout duplicates some provider
configuration during the experiment. This is intentional. It keeps comparison
and rollback simple and prevents a global migration of existing agents.
The standalone package boundary requires generated artifacts, clean-consumer
tests, and cross-language parity gates. These checks add build cost, but they
prevent server implementation details from becoming accidental public API.
## Required proof before general availability
- TypeScript and Rust accept and reject the same protocol fixtures.
- Lost acknowledgements, reconnects, runner restarts, and duplicate commands do
not duplicate events or effects.
- Bootstrap replay, binding mismatch, and cross-company access fail closed.
- Only authorized and production-bound actions appear in discovery.
- Cancellation settles provider, runner, transport, and application state.
- Flag-off direct-adapter runs create no native records and start no runner
process.
- Persisted native runs remain readable and recoverable after the flag changes.
- Codex completes the server to PRP to runnerd to provider to server path.
- Existing adapter compatibility tests remain byte stable where specified.
+332 -3
View File
@@ -1,5 +1,10 @@
# Observability
This document is the Observability contract. It covers the OpenTelemetry
trace path and two local instrumentation contracts; see the
[Telemetry Data Contract](../packages/shared/src/telemetry/README.md) for the
separate first-party event system.
Paperclip ships with **opt-in** OpenTelemetry auto-instrumentation for the
server process. When activated it produces **traces only** — no metrics and no
logs are exported by this integration. The OTel packages are *optional peer
@@ -94,8 +99,332 @@ without tracing — your server stays up.
## Scope
This integration emits **traces only**. Metrics and log exporters are out of
scope and intentionally not configured here. Auto-instrumentations for
`fs`, `dns`, and `net` are disabled by default because they are too chatty
The OpenTelemetry export carries **traces only**. Metrics and log exporters
are out of scope and intentionally not configured here. Auto-instrumentations
for `fs`, `dns`, and `net` are disabled by default because they are too chatty
for this workload; everything else from
`@opentelemetry/auto-instrumentations-node` is on (HTTP, Express, PG, etc.).
This document also holds two local instrumentation contracts: the sandbox
startup trace spans, and the sandbox duplex transport instrumentation. Both
sections follow below.
## Sandbox Startup Trace Spans
Paperclip opens OpenTelemetry spans on the sandbox start path. These spans are
an Observability surface. They are not Paperclip Telemetry events. The
generated telemetry contract does not cover them, so this section is their
canonical contract.
The spans are opt-in. Paperclip exports them only when an OTLP endpoint is
configured. With no endpoint the whole span path is a no-op. Paperclip opens the
spans only for a run that targets a remote sandbox. A local run and an SSH run
stay out of these spans.
Every span attribute uses the closed `paperclip.sandbox.startup.` prefix and
rides a fixed allowlist. A command line, an argument, an environment value, a
file path, program output, or a raw identifier never rides a span. It rides
neither as an attribute nor as an event. The producer bounds each free-form
value:
- A command basename maps to a small known set. Any other value maps to `other`.
- A region maps to a small known set. Any other value maps to `unknown`.
- An image id, a sandbox id, and a lease id ride only as a non-reversible short
hash.
Each numeric attribute is finite. Paperclip omits an attribute when its value is
absent, never a misleading `0`.
### Spans
| Span | Scope | Parent |
| --- | --- | --- |
| `sandbox.startup` | The one root span for a sandbox bring-up. | none (root) |
| `workspace.resolve` | Workspace resolution step. | `sandbox.startup` |
| `codex-home.seed` | Managed-home seed step. | `sandbox.startup` |
| `skills.reconcile` | Skills reconcile step. | `sandbox.startup` |
| `stage.sync` | Workspace stage-sync step. | `sandbox.startup` |
| `snapshot.git` | Host-side git workspace enumeration inside `stage.sync` (`git status --ignored`, the HEAD diffs, `ls-files`). | `stage.sync` |
| `snapshot.baseline` | Host-side baseline workspace content-hash walk inside `stage.sync`, kept for restore. | `stage.sync` |
| `stage.workspace` | One inbound workspace stage task inside `stage.sync`. It packs and uploads the workspace. | `stage.sync` |
| `stage.asset.<key>` | One inbound asset stage task inside `stage.sync`. It packs and uploads one managed-home asset. The `<key>` segment is the asset key. | `stage.sync` |
| `stage.project.<id>` | One inbound referenced-project stage task inside `stage.sync`. It uploads one referenced project. The `<id>` segment is the project id. | `stage.sync` |
| `pack` | Host-side workspace tarball build inside the `stage.workspace` task. | `stage.workspace` |
| `bridge.paperclip` | Paperclip bridge start step. | `sandbox.startup` |
| `bridge.process-session` | Process-session bridge start step. | `sandbox.startup` |
| `acp.handshake` | ACP session handshake step. | `sandbox.startup` |
| `sandbox.syncBack` | The settlement sync-back that restores the managed home at teardown. | the active run span |
| `restore.workspace` | One outbound workspace restore task at teardown. It reads the sandbox workspace back and merges it into the host workspace. | `sandbox.syncBack` |
| `restore.asset.<key>` | One outbound asset restore task at teardown. It reads one asset back to its host store. The `<key>` segment is the asset key. | `sandbox.syncBack` |
| `sandbox.agentSession.sendInput` | One outbound ACP message to the agent — the socket handler's one `writeTextFile` exec. | the active run span |
| `sandbox.agentSession.pollOutput` | One 100 ms poll tick — `list`, then `read`+`remove` per file found (`1 + 2n` execs). | the active run span |
| `sandbox.callbackBridge.relayRequest` | One Paperclip-API callback request — read the request, write the response, remove it. | the active run span |
| `sandbox.agentProcess` | The persistent streamed agent process the process-session bridge launches; open until the process settles or the bridge tears down, whichever comes first. | the active run span |
| `sandbox.exec` | One host-to-sandbox execution. | the active step or wrapper span |
A step span name is the step name. The `sandbox.exec` span parents to the step
span that runs the execution, so each execution nests under its step. Within
`stage.sync`, the host-side sub-steps `snapshot.git` and `snapshot.baseline` open
as child spans of the step, so the host work at the head of the step is
attributed rather than showing as a gap. Each inbound sync operation also opens
its own task span under `stage.sync`: `stage.workspace`, one `stage.asset.<key>`
per asset, and one `stage.project.<id>` per referenced project. The `pack` span
nests under `stage.workspace`, because the host builds the tarball inside that
task. Two concurrent tasks produce overlapping spans.
The settlement `sandbox.syncBack` span runs at teardown and parents to the run
span. It wraps the managed-home restore. Each outbound restore operation opens
its own task span under `sandbox.syncBack`: `restore.workspace` and one
`restore.asset.<key>` per asset. Two concurrent restore tasks produce overlapping
spans. A run-time
`sandbox.exec` span parents instead to the run-time wrapper span that runs it
(`sandbox.agentSession.sendInput`, `sandbox.agentSession.pollOutput`,
`sandbox.callbackBridge.relayRequest`, or `sandbox.agentProcess`). Each run-time
wrapper span parents to the live run span (`agent.turn` during the turn,
`task.run` otherwise). With no active trace context the exec span opens
unparented.
`sandbox.agentProcess` wraps the persistent streamed agent process. The
process-session bridge launches it during `bridge.process-session`, so it opens
under `task.run` — no turn has started yet. It therefore overlaps the sibling
`agent.turn` rather than nesting under it or dangling off the short-lived bring-up
step. The span ends when the process settles or when the bridge tears down,
whichever comes first. The bridge tears down before the run root span ends, so
the span never outlives `task.run` even when the process lingers past teardown
(the sandbox `execute` has no cancel, so a lingering process cannot be forced to
resolve).
The root span sets the error status when the bring-up fails. Each step span sets
the error status when its step fails. The `sandbox.exec` span sets the error
status when the exit code is non-zero or the execution throws.
### Outcome values
The `paperclip.sandbox.startup.outcome` attribute uses a closed value set:
- `ok` — the step or the execution settled with a success result.
- `skipped` — a warm cache skipped the step; the step ran no work.
- `failed` — the step or the execution threw, or the exit code was non-zero.
### Root span attributes
The `sandbox.startup` root span uses this closed attribute allowlist.
| Attribute | Type | Optional | Meaning |
| --- | --- | --- | --- |
| `paperclip.sandbox.startup.root.wall_ms` | number | no | The root-span wall time of the whole bring-up. |
| `paperclip.sandbox.startup.root.work_ms` | number | no | The sum of the step wall times. |
| `paperclip.sandbox.startup.root.diff_ms` | number | no | `work_ms − wall_ms`; the overlap the parallel steps saved. |
| `paperclip.sandbox.startup.provider` | string | yes | The normalized provider family. |
| `paperclip.sandbox.startup.cold_start` | boolean | yes | Whether the bring-up is a cold start. |
| `paperclip.sandbox.startup.region` | string | yes | The clamped region label. |
| `paperclip.sandbox.startup.image_id` | string | yes | The hashed image id. |
| `paperclip.sandbox.startup.sandbox_id` | string | yes | The hashed sandbox id. |
| `paperclip.sandbox.startup.lease_id` | string | yes | The hashed lease id. |
### Step span attributes
Each bring-up step span uses this closed attribute allowlist. The step name
rides the span name, so no `step` attribute repeats it.
| Attribute | Type | Optional | Meaning |
| --- | --- | --- | --- |
| `paperclip.sandbox.startup.step.wall_ms` | number | no | The wall time of the step. |
| `paperclip.sandbox.startup.outcome` | string | no | The step outcome (`ok`, `skipped`, or `failed`). |
| `paperclip.sandbox.startup.provider` | string | yes | The normalized provider family. |
| `paperclip.sandbox.startup.batch` | string | yes | A shared tag that marks two parallel steps as one batch. |
| `paperclip.sandbox.startup.handshake.create_runtime.wall_ms` | number | yes | The create-runtime sub-time of the `acp.handshake` step. |
| `paperclip.sandbox.startup.handshake.ensure_session.wall_ms` | number | yes | The ensure-session sub-time of the `acp.handshake` step. |
The round-trip count and the provider durations no longer ride a step span. The
per-execution `sandbox.exec` child spans carry that detail.
### `sandbox.exec` span attributes
The `sandbox.exec` span uses this closed attribute allowlist. Paperclip omits a
numeric attribute when the provider does not report the value.
| Attribute | Type | Optional | Meaning |
| --- | --- | --- | --- |
| `paperclip.sandbox.startup.provider` | string | no | The normalized provider family. |
| `paperclip.sandbox.startup.exec.command` | string | no | The clamped `argv[0]` command label. |
| `paperclip.sandbox.startup.exec.exit_code` | number | yes | The numeric process exit code. |
| `paperclip.sandbox.startup.exec.wall_ms` | number | no | The host-measured wall time of the execution. |
| `paperclip.sandbox.startup.exec.wait_before_ms` | number | yes | The provider handle-fetch wait before the execution ran. |
| `paperclip.sandbox.startup.exec.sandbox_ms` | number | yes | The in-sandbox run time of the execution. |
| `paperclip.sandbox.startup.exec.network_ms` | number | yes | The transport time the host adds; `wall_ms − wait_before_ms − sandbox_ms`. |
| `paperclip.sandbox.startup.exec.critical_path` | boolean | no | Whether the execution sits on the startup critical path. |
| `paperclip.sandbox.startup.exec.cache_hit` | boolean | yes | Whether the provider served the sandbox handle from its warm cache. |
| `paperclip.sandbox.startup.outcome` | string | no | The execution outcome (`ok` or `failed`). |
The plugin decides the cache hit at the sandbox-handle lookup. The span no
longer infers a cache hit from `wait_before_ms == 0`. Paperclip omits the
`cache_hit` attribute when the provider does not report the value.
To add a span attribute, extend the `SANDBOX_STARTUP_SPAN_ATTRS` allowlist in
the code first. Keep the attribute low-cardinality and free of user content.
### Provider spans
A sandbox provider plugin also opens spans for its own sync steps. These spans
use the `sandbox.daytona.` name prefix. They share the
`paperclip.sandbox.startup.` attribute prefix and obey the same opt-in and
no-user-content rules as the startup spans above.
The plugin worker runs in a separate process from the host. So the host treats
every field of a worker-sent span as untrusted input. The host re-clamps the
span name and every attribute at one boundary, the `span.record` host handler,
before it records the span.
| Span | Scope | Parent |
| --- | --- | --- |
| `sandbox.daytona.pack` | The host-local pack step that builds the upload tarball. It makes no sandbox round trip. | the active startup step span |
| `sandbox.daytona.transfer` | The transfer step: an upload to the sandbox (inbound) or a download from the sandbox (outbound). The `paperclip.sandbox.startup.transfer.direction` attribute records the direction. | the active sync task span (`stage.*` inbound, `restore.*` under `sandbox.syncBack` outbound) |
| `sandbox.daytona.ensureDirectory` | The `mkdir -p` step that ensures a directory exists before a write. | the active startup step span |
| `sandbox.daytona.checkSymlinkEscape` | The re-check step that a path resolves inside the workspace root before use. | the active startup step span |
| `sandbox.daytona.promote` | The atomic move of a staged temp onto its target via a pinned dir handle. | the active startup step span |
| `sandbox.daytona.extractTarball` | The one round trip that re-checks the path, runs `tar -xf`, and removes the scratch tarball. | the active startup step span |
| `sandbox.daytona.postUploadCommand` | One caller-supplied post-upload command. | the active startup step span |
| `sandbox.daytona.session.open` | The create of the one persistent session for a lease, on the first in-run command. | the active run span |
| `sandbox.daytona.session.close` | The delete of that persistent session on lease release. | the active run span |
| `sandbox.daytona.other` | Any span name outside the known set. | the active startup step span |
The host clamps the span name to the closed set of leaf names above (`pack`,
`transfer`, `ensureDirectory`, `checkSymlinkEscape`, `promote`, `extractTarball`,
`postUploadCommand`, `session.open`, and `session.close`). The host maps a known
name to `sandbox.daytona.<name>`. The host maps any other value to
`sandbox.daytona.other`, so a span name never carries free-form data. Only the
daytona provider emits these spans today, so the segment is the literal
`daytona`.
The `sandbox.daytona.*` spans use this closed attribute allowlist. The host
drops every other key, so a command, an argument, a path, an id, a standard
output, or a standard error never rides a provider span. The host records only
the attributes that the producer sends for one span.
| Attribute | Type | Optional | Meaning |
| --- | --- | --- | --- |
| `paperclip.sandbox.startup.provider` | string | no | The normalized provider family. |
| `paperclip.sandbox.startup.outcome` | string | yes | The step outcome (`ok`, `skipped`, or `failed`). |
| `paperclip.sandbox.startup.pack.wall_ms` | number | yes | The host-local wall time of the pack step. It rides the `sandbox.daytona.pack` span. |
| `paperclip.sandbox.startup.transfer.wall_ms` | number | yes | The wall time of the transfer step. It rides the `sandbox.daytona.transfer` span. |
| `paperclip.sandbox.startup.transfer.guard.count` | number | yes | The number of serial guard round trips before one transfer. It rides the `sandbox.daytona.transfer` span. |
| `paperclip.sandbox.startup.transfer.direction` | string | yes | The transfer direction (`inbound` or `outbound`). It rides the `sandbox.daytona.transfer` span. |
The `span.record` host handler enforces the allowlist. It re-maps `provider`
through the provider-family normalizer. It keeps `outcome` only when the value
is `ok`, `skipped`, or `failed`. It keeps `transfer.direction` only when the
value is `inbound` or `outbound`. It keeps a numeric attribute only when the
value is a finite number. It drops a status message and keeps only the numeric
status code. The handler never throws, because observability must not change the
sync control flow.
The `span.record` host method needs the `environment.drivers.register`
capability. So only a plugin that registers an environment driver may emit a
provider span. The capability gate rejects a provider span from any other
plugin.
The host parents each provider span to the active sync task span. An inbound
transfer runs inside a `stage.*` task span, so its provider spans parent there.
An outbound transfer runs inside a `restore.*` task span under `sandbox.syncBack`
at teardown, so its provider spans parent there. The host mints a W3C
`traceparent` from the active task span and passes it to the plugin worker on the
per-call invocation channel. The teardown restore runs inside the run-parented
`sandbox.syncBack` span, so the host mints a `traceparent` for an outbound
provider span the same way it does for an inbound one. The worker tags its span with the
`traceparent` and treats the value as opaque. The worker never derives the
parent from it. The host recovers the `traceparent` from its own invocation
record, so a worker can never forge a parent. The host validates the
`traceparent` and rejects a missing or malformed value. With no active host
trace context the worker sends no span, so the whole provider-span path is a
no-op.
## Sandbox Duplex Transport Instrumentation
This section documents one duplex transport with three sinks: an
OpenTelemetry span, a counter in the `tool_runtime_metric_counters` table, and
one run-log event.
Paperclip opens a fixed observability surface for the sandbox duplex transport.
This instrumentation is separate from Paperclip Telemetry events and from the
sandbox startup trace spans above. The generated telemetry contract does not
cover it, so this section is its canonical contract. The code owner is
`packages/adapter-utils/src/duplex-observability.ts`. That module holds each name and
each enum value as a literal constant, so the surface never drifts.
The surface is opt-in. The host injects a recorder that binds the span to the
OTel tracer, the counter to the guarded counter store in
`server/src/services/tool-runtime-metrics.ts`, and the event to the run-events
bridge. The default recorder is a no-op, so the whole surface stays inert until
the host binds a real recorder. Every recorder call sits inside an error swallow,
so a telemetry failure never breaks the request path.
The surface carries no user content. No route, no query, no request body, no
token, and no raw identifier rides a span, a counter, or an event. Each record
carries only the closed dimension keys below and, for the request span, a
latency. The `provider` dimension carries only the allowlisted public value
`daytona`. Any other plugin key maps to `other` before the record reaches a sink,
so a raw plugin key never reaches a span attribute, a counter label, or an event
field.
### Spans
| Span | Scope | Latency |
| --- | --- | --- |
| `sandbox.duplex.channel_open` | One duplex channel-open attempt. The `outcome` dimension is `ok` when the channel opened and readiness passed, or `error` when the open or readiness failed. | none |
| `sandbox.duplex.request` | One duplex request the broker forwarded to the host. | The request latency in milliseconds. |
### Event
| Event | Scope |
| --- | --- |
| `sandbox.duplex.transport` | The host emits it at each transport boundary: a ready duplex channel, a fallback to the file bridge, and a terminal channel loss. Its dimensions record the boundary. |
### Counters
| Counter | Scope |
| --- | --- |
| `sandbox_duplex_channel_open_total` | One successful duplex channel open. |
| `sandbox_duplex_fallback_total` | One fallback to the file bridge. The `fallback_reason` dimension records the cause. |
| `sandbox_duplex_loss_total` | One terminal duplex channel loss. The `loss_class` dimension records the phase. |
| `sandbox_duplex_session_leak_total` | One leaked provider session at teardown. |
### Aggregate byte ledger metrics
The host aggregate byte ledger owns one process-scoped gauge and two
process-scoped counters. The ledger bounds the retained bytes across every live
duplex route in one process. It sets the gauge on each reserve and each release.
It increments a counter on a rejected reservation and on an accounting defect.
These records carry no dimension label. The guarded counter store keys each
counter on `(companyId, metric)`, and the gauge reports one process value, so no
dynamic dimension rides them. The code owner is
`packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts`, and the metric
names are literal constants in `duplex-observability.ts`.
| Metric | Type | Scope |
| --- | --- | --- |
| `sandbox_duplex_aggregate_bytes_in_use` | gauge | The aggregate retained bytes across every live duplex route. The ledger sets it on each reserve and each release. |
| `sandbox_duplex_aggregate_byte_reservation_rejections_total` | counter | One rejected aggregate byte reservation. The ledger increments it when a reservation would pass the aggregate ceiling. |
| `sandbox_duplex_aggregate_byte_accounting_underflow_total` | counter | One aggregate byte accounting defect. The ledger increments it on a double release or on a transfer of a token it does not hold. |
### Dimension keys
Counters carry no dimension labels. The guarded counter store keys each counter
on `(companyId, metric)` with no label column, so the `fallback_reason` and
`loss_class` values fold into the counter metric name instead. The full closed
dimension set below rides only the spans and the `sandbox.duplex.transport`
event, which use only these closed keys. A test asserts the exact set, so a new
key never reaches a sink by accident.
| Key | Type | Optional | Value set |
| --- | --- | --- | --- |
| `provider` | string | no | `daytona`, or `other` for any other plugin key. |
| `transport` | string | no | `duplex`, `http2`, or `file`. `duplex` names the retired bespoke frame protocol; `http2` names the Node HTTP/2 session over the sandbox channel; a fallback record uses `file`. |
| `outcome` | string | yes | `ok` or `error`. |
| `fallback_reason` | string | yes | `gate_off`, `capability_absent`, `route_busy`, `entrypoint_sync_failed`, `broker_construction_failed`, `channel_open_failed`, `ready_invalid`, `ready_nonce_mismatch`, `ready_timeout`, `contaminated`, `aggregate_bytes_exceeded`, or `preface_missing`. It rides only a fallback record. `route_busy` marks the process-scoped route ceiling full. `entrypoint_sync_failed` and `broker_construction_failed` mark the named build step. `channel_open_failed` marks a failed channel open. `aggregate_bytes_exceeded` marks a readiness handshake, or an `http2` post-preface pre-bind buffer, where the host fell back because the process aggregate byte ceiling had no room. `preface_missing` marks a missing or an invalid HTTP/2 client connection preface inside the bounded readiness buffer: the host found no valid preface after the accepted READY line, aborted the `http2` open, and moved the run to the file bridge (`queue_v1`) one time. |
| `loss_class` | string | yes | `pre_dispatch` or `post_dispatch`, relative to the first request dispatch. It rides only a loss record. |
| `loss_reason` | string | yes | `stdin_eof`, `provider_exit`, `heartbeat_timeout`, `rpc_failure`, `write_error`, `transport_closed`, or `other`. The host maps every loss cause to one of these values, so no raw provider text reaches a sink. `write_error` marks a rejected host-to-sandbox write. `transport_closed` marks a reason-less provider transport close with no exit data. It rides only a loss record. |
To add a name or an enum value, extend the literal constant in
`duplex-observability.ts` first, then update the test that asserts the closed set.
Keep every dimension low-cardinality and free of user content.
+96
View File
@@ -0,0 +1,96 @@
# Run-Log Events
Run-log events write to the `heartbeat_run_events` table
(`packages/db/src/schema/heartbeat_run_events.ts:6-20`). They are not
Paperclip Telemetry events, and they are not OpenTelemetry exports. A run-log
event needs no operator endpoint.
## Native PRP Run-Log Events
The hidden native coordinator writes each validated PRP event to the bound
run's existing event stream before it acknowledges the runner. The row keeps
the PRP `eventType`, source instance, source event ID, source sequence, protocol
schema version, and a SHA-256 digest of the canonical source envelope. Its
payload is `{ "prpEvent": <canonical PRP event> }`.
The writer locks the native `heartbeat_runs` row and allocates the existing
per-run `seq` cursor. A byte-equivalent retry reuses the first row; a changed
retry or source-sequence gap is rejected. Company, issue, agent, run, session,
and runner-source bindings must match the persisted native run. Bootstrap
tickets, reconnect leases, authentication proofs, encryption keys, and raw
credential material are never written to the run log.
These records remain run-log events. They do not create an OpenTelemetry or
Paperclip Telemetry export, and legacy adapters do not use this writer.
## Sandbox Startup Run-Log Event
Paperclip writes one `run.startup.step` event to the run log for each bring-up
step. This event is a run-log record, not a first-party telemetry event. The
generated telemetry contract does not cover it, so this section is its canonical
contract.
The event payload carries only three fields.
| Field | Type | Meaning |
| --- | --- | --- |
| `step` | string | The bring-up step name, for example `stage.sync`. |
| `durationMs` | number | The wall time of the step. A skipped step reports `0`. |
| `outcome` | string | The step outcome (`ok`, `skipped`, or `failed`). |
The event no longer carries the per-step round-trip count or the provider
duration fields. It dropped `roundTrips`, `providerExecMs`, `providerGetMs`,
`createRuntimeMs`, and `ensureSessionMs`. The startup spans in
[`doc/observability.md`](observability.md) carry that detail now. The
`sandbox.exec` child spans hold the round-trip and provider durations. The
`acp.handshake` step span holds the create-runtime and ensure-session
sub-times.
To read the detailed timing, use the startup spans. The spans need an OTLP
endpoint. A run with no endpoint keeps only the three run-log fields above.
## Run Phase Timing Run-Log Event
Paperclip writes one `run.phase.timing` event to the run log for each
run-lifecycle phase. This event is a run-log record, not a first-party telemetry
event. The generated telemetry contract does not cover it, so this section is its
canonical contract. The producer is `emitRunPhaseTiming` in
`packages/adapter-utils/src/acpx-engine/startup-timing.ts`.
The event payload carries only three fields.
| Field | Type | Meaning |
| --- | --- | --- |
| `phase` | string | The run-lifecycle phase name from the closed allowlist below. |
| `durationMs` | number | The wall time of the phase. A negative or a non-finite value clamps to `0`. |
| `outcome` | string | The phase outcome (`ok` or `failed`). |
The `phase` field is one member of a closed, low-cardinality allowlist. The
producer drops any event whose phase name is outside this allowlist, so a
free-form label never reaches the run log. The allowlist has twelve phase names.
| Phase | Meaning |
| --- | --- |
| `place_workspace` | Place the run workspace. |
| `start_transport` | Start the agent transport. |
| `create_runtime` | Create the agent runtime. |
| `ensure_session` | Ensure the agent session exists. |
| `configure_session` | Configure the agent session. |
| `prepare_turn` | Prepare the turn. |
| `turn` | Run the turn. |
| `end_session` | End the agent session. |
| `settle_reuse` | Settle the session for reuse. |
| `stop_transport` | Stop the agent transport. |
| `sync_back` | Sync the workspace back. |
| `release_staging_lease` | Release the staging lease. |
The payload never carries a command, an argument, a path, an environment value,
or a raw identifier. The event rides the `ctx.onEvent` run-event bridge and is
run-log-only. It needs no OTLP endpoint.
## Related instrumentation
The sandbox duplex transport also writes one run-log event as one of its three
sinks. See the
[Sandbox Duplex Transport Instrumentation](observability.md#sandbox-duplex-transport-instrumentation)
section in the Observability contract.
+5
View File
@@ -11,6 +11,11 @@ services:
build:
context: ..
dockerfile: Dockerfile
# Ceiling on processes. The image makes tini PID 1 so orphaned agent
# descendants are reaped instead of piling up as zombies; this is the
# backstop if something leaks anyway — the container hits its own limit and
# dies visibly rather than starving the host of pids.
pids_limit: 2048
ports:
- "${PAPERCLIP_PORT:-3100}:3100"
environment:
+9
View File
@@ -19,6 +19,15 @@ services:
build:
context: ..
dockerfile: Dockerfile
# No `init: true` here on purpose. The image makes tini PID 1 itself
# (Dockerfile ENTRYPOINT), which covers every launch path -- compose, plain
# `docker run`, the quadlet units and the ECS task definition -- instead of
# one lever per orchestrator. Setting it here too would nest docker-init
# around tini and make tini warn that it is not PID 1 on every boot.
# pids_limit is the backstop: if something ever does leak processes again,
# this container hits its own ceiling and dies visibly rather than starving
# the whole host of pids.
pids_limit: 2048
ports:
- "3100:3100"
environment:
+2 -2
View File
@@ -54,7 +54,7 @@
"smoke:mcp-fixtures": "node scripts/smoke/mcp-fixture-harness.mjs",
"smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh",
"smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs",
"test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs",
"test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs",
"storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs",
"test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts",
"test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack",
@@ -72,7 +72,7 @@
"devDependencies": {
"@playwright/test": "^1.62.1",
"cross-env": "^10.1.0",
"esbuild": "^0.28.1",
"esbuild": "^0.28.2",
"typescript": "^5.7.3",
"vitest": "^4.1.10"
},
@@ -36,8 +36,6 @@ import {
type AcpxEngineExecutorOptions,
} from "./execute.js";
import { runChildProcess } from "../server-utils.js";
import { createDuplexBridgeBroker } from "../duplex-bridge-broker.js";
import type { CommandManagedDuplexChannel } from "../command-managed-runtime.js";
import {
getActiveStepContext,
runWithRuntimeParent,
@@ -5734,56 +5732,55 @@ describe("ACPX engine run lifecycle corrections (F3: one teardown error policy)"
});
});
describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => {
describe("ACPX engine sandbox bridge run-disposition seam (fail-closed)", () => {
beforeEach(() => {
vi.clearAllMocks();
});
// A minimal in-memory duplex channel. The test drives a channel exit to latch a
// real loss in a real broker, so the seam reads a real run disposition.
function createFakeDuplexChannel() {
let exitListener: ((exit: { exitCode: number | null }) => void) | null = null;
const channel: CommandManagedDuplexChannel = {
write(): void {},
onData(): void {},
onExit(listener: (exit: { exitCode: number | null }) => void): void {
exitListener = listener;
},
stop(): void {},
close(): Promise<void> {
return Promise.resolve();
},
};
return {
channel,
emitExit: (exit: { exitCode: number | null }) => exitListener?.(exit),
};
}
// Wrap a started real broker in a paperclip bridge handle. The handle exposes
// the same run-disposition surface the sandbox bridge exposes, so the seam runs
// against the real latch and the real orderly-completion mark.
function bridgeOverBroker(fake: ReturnType<typeof createFakeDuplexChannel>) {
const broker = createDuplexBridgeBroker({
channel: fake.channel,
forwardRequest: async () => ({ status: 200 }),
/**
* A minimal run-disposition latch. It reproduces the same ordering rule the
* bridge transport applies: the first ordered loss or orderly completion
* latches the terminal disposition, and a later call never overrides it.
* The test drives the latch directly, with no channel and no live process.
*/
function createFakeBridgeHandle() {
let lossOrdered = false;
let lossReason: string | null = null;
let completionOrdered = false;
const readDisposition = () => ({ failed: lossOrdered, lossReason });
const markOrderlyCompletion = vi.fn(() => {
if (completionOrdered || lossOrdered) return;
completionOrdered = true;
});
const settleRunDisposition = vi.fn(() => {
markOrderlyCompletion();
return readDisposition();
});
broker.start();
const markOrderlyCompletion = vi.fn(() => broker.markOrderlyCompletion());
const settleRunDisposition = vi.fn(() => broker.settleRunDisposition());
const stop = vi.fn(async () => {});
const handle = {
env: {
PAPERCLIP_API_URL: "http://127.0.0.1:1",
PAPERCLIP_API_KEY: "bridge-token",
PAPERCLIP_API_BRIDGE_MODE: "duplex_v1",
PAPERCLIP_API_BRIDGE_MODE: "http2_v1",
},
readRunDisposition: () => broker.runDisposition,
readRunDisposition: () => readDisposition(),
settleRunDisposition,
markOrderlyCompletion,
stop,
};
return { broker, handle, markOrderlyCompletion, settleRunDisposition };
return {
handle,
markOrderlyCompletion,
settleRunDisposition,
readDisposition,
// Record the first ordered loss. A loss ordered after a completion, or a
// second loss, is a no-op — the same rule the real transport applies.
emitLoss: (reason: string) => {
if (lossOrdered || completionOrdered) return;
lossOrdered = true;
lossReason = reason;
},
};
}
// A runtime whose one turn completes cleanly. The `beforeResult` hook runs at
@@ -5888,14 +5885,13 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () =>
} as never);
}
it("fails a completed run when the duplex channel was lost before the completion", async () => {
it("fails a completed run when the bridge channel was lost before the completion", async () => {
const sandbox = await setupRemoteSandbox();
const fake = createFakeDuplexChannel();
const { broker, handle, settleRunDisposition } = bridgeOverBroker(fake);
const fake = createFakeBridgeHandle();
// Latch the loss before the ACP terminal resolves.
const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 }));
const runtime = runtimeWithControlledResult(() => fake.emitLoss("provider_exit"));
const result = await runRemote(handle, runtime, sandbox);
const result = await runRemote(fake.handle, runtime, sandbox);
// The lost channel overrides the nominally completed terminal to a failure.
expect(result.exitCode).not.toBe(0);
@@ -5905,65 +5901,62 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () =>
expect(result.resultJson).toMatchObject({ status: "failed" });
// The seam read the disposition through the atomic settle step, and the
// latched loss kept the failure, so no orderly completion ordered.
expect(settleRunDisposition).toHaveBeenCalledTimes(1);
expect(broker.runDisposition.failed).toBe(true);
expect(fake.settleRunDisposition).toHaveBeenCalledTimes(1);
expect(fake.readDisposition().failed).toBe(true);
});
it("keeps a completed run a success when the channel stays live, and a later teardown loss is benign", async () => {
const sandbox = await setupRemoteSandbox();
const fake = createFakeDuplexChannel();
const { broker, handle, settleRunDisposition } = bridgeOverBroker(fake);
const fake = createFakeBridgeHandle();
// No loss before the completion.
const runtime = runtimeWithControlledResult();
const result = await runRemote(handle, runtime, sandbox);
const result = await runRemote(fake.handle, runtime, sandbox);
expect(result.exitCode).toBe(0);
expect(result.errorCode ?? null).toBeNull();
// The atomic settle step marked the orderly completion for the
// success-eligible terminal.
expect(settleRunDisposition).toHaveBeenCalledTimes(1);
expect(fake.settleRunDisposition).toHaveBeenCalledTimes(1);
// A teardown loss ordered after the orderly completion is a normal teardown,
// so the run disposition stays a success.
fake.emitExit({ exitCode: 0 });
expect(broker.runDisposition.failed).toBe(false);
fake.emitLoss("provider_exit");
expect(fake.readDisposition().failed).toBe(false);
});
it("does not let a later completion or activity clear the loss latch", async () => {
const sandbox = await setupRemoteSandbox();
const fake = createFakeDuplexChannel();
const { broker, handle } = bridgeOverBroker(fake);
const fake = createFakeBridgeHandle();
// Latch the loss before the ACP terminal resolves.
const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 }));
const runtime = runtimeWithControlledResult(() => fake.emitLoss("provider_exit"));
const result = await runRemote(handle, runtime, sandbox);
const result = await runRemote(fake.handle, runtime, sandbox);
expect(result.exitCode).not.toBe(0);
expect(result.errorCode).toBe("duplex_channel_lost");
// A later orderly-completion mark and further channel activity cannot clear
// the latched loss.
broker.markOrderlyCompletion();
fake.emitExit({ exitCode: 0 });
expect(broker.runDisposition.failed).toBe(true);
expect(broker.runDisposition.lossReason).toBe("provider_exit");
fake.markOrderlyCompletion();
fake.emitLoss("provider_exit");
expect(fake.readDisposition().failed).toBe(true);
expect(fake.readDisposition().lossReason).toBe("provider_exit");
});
it("marks an orderly completion on a failed terminal so the teardown loss emits no false loss", async () => {
const sandbox = await setupRemoteSandbox();
const fake = createFakeDuplexChannel();
const { broker, handle, markOrderlyCompletion } = bridgeOverBroker(fake);
const fake = createFakeBridgeHandle();
// The turn fails, and no channel loss ordered before the finalization.
const runtime = runtimeWithFailedResult();
const result = await runRemote(handle, runtime, sandbox);
const result = await runRemote(fake.handle, runtime, sandbox);
// The failed terminal stays a failure, but not a duplex loss.
// The failed terminal stays a failure, but not a bridge-channel loss.
expect(result.exitCode).not.toBe(0);
expect(result.errorCode).not.toBe("duplex_channel_lost");
// The non-success-eligible terminal marked the orderly completion, so the
// teardown channel_exit orders after the mark and does not latch a loss.
expect(markOrderlyCompletion).toHaveBeenCalledTimes(1);
fake.emitExit({ exitCode: 0 });
expect(broker.runDisposition.failed).toBe(false);
// teardown loss orders after the mark and does not latch a loss.
expect(fake.markOrderlyCompletion).toHaveBeenCalledTimes(1);
fake.emitLoss("provider_exit");
expect(fake.readDisposition().failed).toBe(false);
});
});
@@ -15,7 +15,7 @@ import type {
import {
adapterExecutionTargetSessionIdentity,
describeAdapterExecutionTarget,
adapterExecutionTargetDuplexTelemetryRecorder,
adapterExecutionTargetDuplexObservabilityRecorder,
adapterExecutionTargetEnablesSandboxDuplexBridge,
formatAdapterExecutionTimeoutErrorMessage,
formatAdapterExecutionTimeoutStartLogLine,
@@ -33,8 +33,8 @@ import {
type PreparedAdapterExecutionTargetRuntime,
type SandboxAdditionalSource,
} from "@paperclipai/adapter-utils/execution-target";
import type { DuplexLossReason } from "../duplex-telemetry.js";
import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "../duplex-bridge-broker.js";
import type { DuplexLossReason } from "../duplex-observability.js";
import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "../bridge-transport-contract.js";
import {
DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE,
applyPaperclipWorkspaceEnv,
@@ -2052,7 +2052,7 @@ async function buildRuntime(input: {
timeoutSec,
hostApiToken: env.PAPERCLIP_API_KEY,
enableSandboxDuplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(remoteTarget),
duplexTelemetryRecorder: adapterExecutionTargetDuplexTelemetryRecorder(remoteTarget),
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(remoteTarget),
onLog: input.ctx.onLog,
getRuntimeParentContext: input.getRuntimeParentContext,
runtimeSpan: input.runtimeSpan,
@@ -3390,10 +3390,11 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) {
.onLog("stderr", `[paperclip] ACPX teardown step "${step}" failed: ${reason}\n`)
.catch(() => {});
};
// Emit one per-phase timing telemetry event. It is observability-only: it
// carries the phase name (from the closed allowlist), the wall time, and the
// outcome, and never a command, a path, an environment value, or an
// identifier. Telemetry failure never fails the run.
// Emit one per-phase timing run-log event. It is not an OpenTelemetry
// export and it is not a Telemetry event: it carries the phase name
// (from the closed allowlist), the wall time, and the outcome, and
// never a command, a path, an environment value, or an identifier. A
// failure to emit this event never fails the run.
const emitPhase = (phase: string, startMs: number, outcome: "ok" | "failed"): Promise<void> =>
emitRunPhaseTiming(ctx, phase, now() - startMs, outcome);
// Time a settlement step and emit its phase timing on every path. A step
@@ -43,7 +43,7 @@ export function normalizeProviderFamily(key: string | undefined): string {
/**
* The common prefix for every sandbox-startup span attribute. One prefix keeps
* the attribute namespace closed and easy to find in the telemetry backend.
* the attribute namespace closed and easy to find in the OpenTelemetry backend.
*/
export const SANDBOX_STARTUP_SPAN_ATTR_PREFIX = "paperclip.sandbox.startup.";
@@ -794,11 +794,12 @@ export async function emitSkippedStartupStep(
/**
* Structured event emitted once per named run-lifecycle phase, so the duration
* and the outcome of each phase land in the run-events stream. It is
* observability-only and rides the existing `ctx.onEvent` bridge. The payload is
* a closed shape: exactly `phase`, `durationMs`, and `outcome`. The phase name is
* from a closed allowlist, so the event never carries a command, an argument, a
* path, an environment value, or a raw identifier.
* and the outcome of each phase land in the run-events stream. It is a
* run-log event and rides the existing `ctx.onEvent` bridge. It never changes
* startup control flow. The payload is a closed shape: exactly `phase`,
* `durationMs`, and `outcome`. The phase name is from a closed allowlist, so
* the event never carries a command, an argument, a path, an environment
* value, or a raw identifier.
*/
export const RUN_PHASE_TIMING_EVENT_TYPE = "run.phase.timing";
@@ -0,0 +1,61 @@
/**
* Shared contract for the sandbox callback bridge transports.
*
* The retired duplex_v1 broker first defined these symbols. The host
* broker is gone, but the http2_v1 transport and the ACPX engine
* run-disposition seam still use them. This leaf module holds the
* survivors, so a caller of the run-disposition seam does not import the
* whole HTTP/2 bridge server module graph to reach one error code.
*/
import type { DuplexLossReason } from "./duplex-observability.js";
/**
* The typed error code the host reports when the bridge control channel
* died before an orderly completion. Both the ACP lane and the CLI lane
* report this one code, so the run disposition is identical across the two
* lanes.
*/
export const DUPLEX_CHANNEL_LOST_ERROR_CODE = "duplex_channel_lost";
/**
* The terminal run disposition a bridge transport computes from its ordered
* lifecycle. A `failed` disposition means a terminal loss ordered before an
* orderly completion, so the run must not report success. The typed loss
* reason names the cause; it is `null` for a success.
*/
export interface DuplexBrokerRunDisposition {
/** True when a terminal loss ordered before an orderly completion. */
failed: boolean;
/** The typed, closed loss reason on a failure; `null` on a success. */
lossReason: DuplexLossReason | null;
}
/** The nested timeout budgets. Each inner budget is smaller than its outer budget. */
export interface DuplexBrokerBudgets {
/** The deadline for one forward call, in milliseconds. */
forwardTimeoutMs: number;
/** The deadline for the broker to send one response frame, in milliseconds. */
responseBudgetMs: number;
/** The deadline the in-sandbox gateway waits for the response frame, in milliseconds. */
gatewayWaitMs: number;
}
/** The default nested budgets: forward 30 s, response 32 s, gateway wait 35 s. */
export const DEFAULT_DUPLEX_BROKER_BUDGETS: DuplexBrokerBudgets = {
forwardTimeoutMs: 30_000,
responseBudgetMs: 32_000,
gatewayWaitMs: 35_000,
};
/**
* The safe HTTP methods. RFC 7231 section 4.2.1 defines this set. A safe method
* does not change host state, so the host applies no mutation for it. A caller
* can retry a safe method after a forward failure without a double-apply risk.
*/
const SAFE_BRIDGE_METHODS = new Set(["GET", "HEAD", "OPTIONS", "TRACE"]);
/** Report whether the method is safe, so a forward failure stays retryable. */
export function isSafeBridgeMethod(method: string): boolean {
return SAFE_BRIDGE_METHODS.has(method.trim().toUpperCase());
}
@@ -8,11 +8,34 @@ import { afterEach, describe, expect, it } from "vitest";
import {
createCommandManagedRuntimeClient,
prepareCommandManagedRuntime,
type CommandManagedDuplexChannel,
type CommandManagedRuntimeRunner,
} from "./command-managed-runtime.js";
import type { SandboxSyncOperation } from "./sandbox-managed-runtime.js";
import type { RunProcessResult } from "./server-utils.js";
/**
* An in-memory fake `CommandManagedDuplexChannel`. It holds no real process; a
* write echoes straight to the one registered data listener, so a test proves
* the channel contract carries a `Uint8Array` chunk with no string coercion in
* between. The fake never exits on its own; a test calls the listener it
* registers with `onExit` only when it needs one.
*/
function createFakeEchoDuplexChannel(): CommandManagedDuplexChannel {
let dataListener: ((chunk: Uint8Array) => void) | null = null;
return {
write(data: Uint8Array): void {
dataListener?.(data);
},
onData(listener: (chunk: Uint8Array) => void): void {
dataListener = listener;
},
onExit(): void {},
stop(): void {},
close: async (): Promise<void> => {},
};
}
const execFile = promisify(execFileCallback);
interface SpawnRunnerHandle {
@@ -1057,4 +1080,22 @@ describe("command managed runtime", () => {
/stdout: tar: workspace-download\.tar: Cannot open: Permission denied/,
);
});
it("test_channel_round_trips_all_byte_values", () => {
const channel = createFakeEchoDuplexChannel();
const allByteValues = Uint8Array.from({ length: 256 }, (_, value) => value);
const received: Uint8Array[] = [];
channel.onData((chunk) => {
received.push(chunk);
});
channel.write(allByteValues);
expect(received).toHaveLength(1);
// A byte value of zero must survive. A UTF-8 string channel loses it: a
// JavaScript string can hold the code point U+0000, but a C-style consumer
// downstream of a string channel often treats it as a terminator.
expect(received[0]).toEqual(allByteValues);
expect(Array.from(received[0] ?? [])).toEqual(Array.from({ length: 256 }, (_, value) => value));
});
});
@@ -37,9 +37,9 @@ export interface DuplexChannelOpenInput {
*/
export interface CommandManagedDuplexChannel {
/** Writes raw input bytes to the channel. */
write(data: string): void;
/** Registers the one data listener. The channel streams each raw chunk in order. */
onData(listener: (chunk: string) => void): void;
write(data: Uint8Array): void;
/** Registers the one data listener. The channel streams each raw byte chunk in order. */
onData(listener: (chunk: Uint8Array) => void): void;
/**
* Registers the one exit listener. The channel calls it one time with the exit.
* A numeric `exitCode` is a real process exit. `transportClosed` is true when the
@@ -117,6 +117,8 @@ export interface CommandManagedRuntimeRunner {
* bidirectional channel to a long-lived command in the sandbox. The SSH runner
* and every provider without the capability omit the member, so a caller gates
* on its presence in the same style as {@link syncIn}/{@link syncOut}.
*
* HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback.
*/
openDuplexChannel?(input: DuplexChannelOpenInput): Promise<CommandManagedDuplexChannel>;
}
@@ -163,6 +163,8 @@ describe("DuplexAggregateByteLedger", () => {
"decoder_buffer",
"readiness_buffer",
"readiness_replay",
"http2_preface_scan",
"http2_preface_replay",
"pending_write",
"stdin_write",
]);
@@ -34,7 +34,7 @@ import {
DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL,
DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL,
DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE,
} from "./duplex-telemetry.js";
} from "./duplex-observability.js";
/**
* The public rejection marker. A retention site that cannot reserve its bytes
@@ -80,6 +80,17 @@ export const MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX = 2 * 1024 * 1024 * 1024;
* frame decoder charges its own retention. This is a separate retention from
* `readiness_buffer`: the gate drops the whole pre-READY buffer on READY, then
* charges only the retained suffix and each later pre-bind chunk under this owner.
* - `http2_preface_scan`: the raw untrusted bytes the `http2_v1` preface scan
* retains while it searches for the client connection preface, from the
* readiness-replay handoff until the preface match or the scan cap. This
* is a separate retention from `readiness_replay`: it starts only after
* the readiness gate hands its own retained suffix to the preface scan.
* - `http2_preface_replay`: the bytes the `http2_v1` preface scan holds after the
* client connection preface, from the preface match until the HTTP/2 server
* binds its downstream listener. This is a separate retention from
* `http2_preface_scan`: the scan drops its own buffer on the preface match,
* then charges only the retained suffix and each later pre-bind chunk under
* this owner.
* - `pending_write`: the raw host-to-worker write payload a pending duplex write
* RPC retains, from the enqueue seam until the RPC settles.
* - `stdin_write`: the serialized host-to-worker frame the child-stdin transport
@@ -99,6 +110,8 @@ export const DUPLEX_AGGREGATE_TOKEN_OWNERS = [
"decoder_buffer",
"readiness_buffer",
"readiness_replay",
"http2_preface_scan",
"http2_preface_replay",
"pending_write",
"stdin_write",
] as const;
@@ -0,0 +1,306 @@
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { DUPLEX_FRAME_VERSION, type DuplexBodyChunkFrame } from "./duplex-frame-codec.js";
import {
DuplexBodyReceiver,
type DuplexBodyReceiverConfig,
splitBodyIntoChunkFrames,
sweepStaleSpillDirectories,
} from "./duplex-body-spool.js";
// Read the whole reassembled body back, whatever path it took. A memory body
// returns its buffer; a spilled body streams from disk. The test uses this only
// to assert the round-trip content, never on a hot path.
async function readBodyBytes(body: {
spilled: boolean;
createReadStream: () => NodeJS.ReadableStream;
}): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of body.createReadStream()) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
let spillRoot: string;
beforeEach(async () => {
spillRoot = await fs.mkdtemp(path.join(os.tmpdir(), "duplex-spool-test-"));
});
afterEach(async () => {
await fs.rm(spillRoot, { recursive: true, force: true }).catch(() => undefined);
});
function makeReceiver(overrides: DuplexBodyReceiverConfig = {}): Promise<DuplexBodyReceiver> {
return DuplexBodyReceiver.create({ spillRootDir: spillRoot, ...overrides });
}
const RAW = 8;
// Feed a whole body through a receiver as an envelope plus chunk frames. Return
// the settled body handle. The caller pushes the chunks the split produces.
async function transfer(
receiver: DuplexBodyReceiver,
id: string,
body: Buffer,
): ReturnType<DuplexBodyReceiver["begin"]> {
const promise = receiver.begin(id, body.length);
const frames = splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION, RAW);
for (const frame of frames) {
const result = receiver.pushChunk(frame);
expect(result.ok).toBe(true);
}
return promise;
}
describe("duplex body reassembler — round trip", () => {
it("reassembles a small body on the memory path", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
const source = Buffer.from("hello duplex body");
const body = await transfer(receiver, "req-mem", source);
expect(body.spilled).toBe(false);
expect(body.byteCount).toBe(source.length);
expect(body.readMemory().equals(source)).toBe(true);
expect((await readBodyBytes(body)).equals(source)).toBe(true);
await body.dispose();
});
it("reassembles a large body on the spill path without holding it in memory", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 16, rawChunkBytes: RAW });
const source = Buffer.from("abcdefghijklmnopqrstuvwxyz0123456789ABCD"); // 40 bytes
const body = await transfer(receiver, "req-spill", source);
expect(body.spilled).toBe(true);
expect(body.byteCount).toBe(source.length);
// A spilled body never loads the whole body into memory on the receive side.
expect(() => body.readMemory()).toThrow();
expect(receiver.spillUsedBytes).toBe(source.length);
expect(receiver.spillOpenFiles).toBe(1);
expect((await readBodyBytes(body)).equals(source)).toBe(true);
await body.dispose();
// The dispose path unlinks the file and releases the reservation.
expect(receiver.spillUsedBytes).toBe(0);
expect(receiver.spillOpenFiles).toBe(0);
const entries = await fs.readdir(receiver.spillDir);
expect(entries).toHaveLength(0);
await receiver.destroy();
});
it("completes a zero-length body at once and accepts no chunk", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 16, rawChunkBytes: RAW });
const body = await receiver.begin("req-zero", 0);
expect(body.byteCount).toBe(0);
expect(body.readMemory().length).toBe(0);
const stray = receiver.pushChunk({
version: DUPLEX_FRAME_VERSION,
type: "body_chunk",
id: "req-zero",
seq: 0,
data: Buffer.from("x").toString("base64"),
});
expect(stray.ok).toBe(false);
if (!stray.ok) expect(stray.error.code).toBe("zero_body_has_chunk");
await body.dispose();
});
});
describe("duplex body reassembler — rejection matrix", () => {
function chunk(id: string, seq: number, data: string): DuplexBodyChunkFrame {
return { version: DUPLEX_FRAME_VERSION, type: "body_chunk", id, seq, data };
}
it("rejects a duplicate seq", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
void receiver.begin("d", 16);
expect(receiver.pushChunk(chunk("d", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true);
const dup = receiver.pushChunk(chunk("d", 0, Buffer.alloc(RAW, 2).toString("base64")));
expect(dup.ok).toBe(false);
if (!dup.ok) expect(dup.error.code).toBe("seq_out_of_order");
});
it("rejects a gap in the seq order", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
void receiver.begin("g", 24);
expect(receiver.pushChunk(chunk("g", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true);
const gap = receiver.pushChunk(chunk("g", 2, Buffer.alloc(RAW, 2).toString("base64")));
expect(gap.ok).toBe(false);
if (!gap.ok) expect(gap.error.code).toBe("seq_out_of_order");
});
it("rejects a reordered seq", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
void receiver.begin("r", 24);
const reorder = receiver.pushChunk(chunk("r", 1, Buffer.alloc(RAW, 1).toString("base64")));
expect(reorder.ok).toBe(false);
if (!reorder.ok) expect(reorder.error.code).toBe("seq_out_of_order");
});
it("rejects non-canonical base64", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
const begun = receiver.begin("b", 8);
begun.catch(() => undefined);
// "AB==" carries non-zero bits in the final sextet, so it decodes to one
// byte and re-encodes to "AA==". The round-trip mismatch marks it not
// canonical.
const bad = receiver.pushChunk(chunk("b", 0, "AB=="));
expect(bad.ok).toBe(false);
if (!bad.ok) expect(bad.error.code).toBe("chunk_not_canonical_base64");
});
it("rejects a short non-final chunk (truncation)", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
void receiver.begin("t", 24);
// A non-final chunk must be exactly RAW bytes. A 4-byte chunk that does not
// complete the 24-byte body is a truncation.
const truncation = receiver.pushChunk(chunk("t", 0, Buffer.alloc(4, 1).toString("base64")));
expect(truncation.ok).toBe(false);
if (!truncation.ok) expect(truncation.error.code).toBe("chunk_wrong_size");
});
it("rejects an empty non-final chunk", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
void receiver.begin("e", 16);
const empty = receiver.pushChunk(chunk("e", 0, ""));
expect(empty.ok).toBe(false);
if (!empty.ok) expect(empty.error.code).toBe("empty_chunk");
});
it("rejects an overrun past the declared body size", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
// A 12-byte body takes one full RAW chunk, then a final chunk of 4 bytes. A
// second full RAW chunk overruns the declared size.
void receiver.begin("o", 12);
expect(receiver.pushChunk(chunk("o", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true);
const overrun = receiver.pushChunk(chunk("o", 1, Buffer.alloc(RAW, 2).toString("base64")));
expect(overrun.ok).toBe(false);
if (!overrun.ok) expect(overrun.error.code).toBe("body_overrun");
});
it("rejects an over-size final chunk", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
void receiver.begin("os", 40);
const over = receiver.pushChunk(chunk("os", 0, Buffer.alloc(RAW + 4, 1).toString("base64")));
expect(over.ok).toBe(false);
if (!over.ok) expect(over.error.code).toBe("chunk_wrong_size");
});
it("rejects a chunk that arrives with no matching envelope", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
const orphan = receiver.pushChunk(chunk("missing", 0, Buffer.alloc(RAW, 1).toString("base64")));
expect(orphan.ok).toBe(false);
if (!orphan.ok) expect(orphan.error.code).toBe("chunk_without_envelope");
});
it("rejects a chunk that arrives after completion", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW });
const promise = receiver.begin("c", RAW);
expect(receiver.pushChunk(chunk("c", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true);
await promise;
const late = receiver.pushChunk(chunk("c", 1, Buffer.alloc(RAW, 2).toString("base64")));
expect(late.ok).toBe(false);
if (!late.ok) expect(late.error.code).toBe("chunk_after_completion");
});
});
describe("duplex body reassembler — spill caps fail closed", () => {
it("fails closed when a write exceeds the per-channel byte cap and unlinks the partial file", async () => {
// A 5-byte cap admits the first RAW=8 chunk write? No: reserve 8 > cap 5, so
// the first write fails closed. Use a cap that admits one chunk but not two.
const receiver = await makeReceiver({
spillThresholdBytes: 4,
rawChunkBytes: RAW,
channelSpillByteCap: RAW + 2,
});
const promise = receiver.begin("cap", 24);
expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "cap", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true);
// The second chunk write reserves past the byte cap.
receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "cap", seq: 1, data: Buffer.alloc(RAW, 2).toString("base64") });
await expect(promise).rejects.toMatchObject({ code: "spill_cap_exceeded" });
// The partial file is gone and the reservation released.
expect(receiver.spillUsedBytes).toBe(0);
expect(receiver.spillOpenFiles).toBe(0);
const entries = await fs.readdir(receiver.spillDir);
expect(entries).toHaveLength(0);
await receiver.destroy();
});
it("fails closed when the open-file cap is reached", async () => {
const receiver = await makeReceiver({
spillThresholdBytes: 4,
rawChunkBytes: RAW,
channelSpillFileCap: 1,
});
// The first spill body claims the one file slot and stays open. The channel
// dispose rejects this in-flight body, so attach the catch at once.
const first = receiver.begin("f1", 24);
first.catch(() => undefined);
expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "f1", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true);
// The second spill body cannot claim a file slot.
await expect(receiver.begin("f2", 24)).rejects.toMatchObject({ code: "spill_cap_exceeded" });
expect(receiver.spillOpenFiles).toBe(1);
// The dispose of the in-flight first body rejects its begin promise with a
// terminal channel-closed error and frees the file slot.
await receiver.disposeBody("f1");
await expect(first).rejects.toMatchObject({ code: "channel_closed" });
expect(receiver.spillOpenFiles).toBe(0);
await receiver.destroy();
});
});
describe("duplex body reassembler — lifecycle cleanup", () => {
it("releases the reservation and unlinks the file on channel death", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 4, rawChunkBytes: RAW });
// The channel death rejects this in-flight body, so attach the catch at once.
const dying = receiver.begin("death", 24);
dying.catch(() => undefined);
expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "death", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true);
// Wait for the write to land, so the file exists before the channel dies.
await new Promise((resolve) => setTimeout(resolve, 20));
expect(receiver.spillOpenFiles).toBe(1);
await receiver.disposeAll();
await expect(dying).rejects.toMatchObject({ code: "channel_closed" });
expect(receiver.spillUsedBytes).toBe(0);
expect(receiver.spillOpenFiles).toBe(0);
const entries = await fs.readdir(receiver.spillDir);
expect(entries).toHaveLength(0);
await receiver.destroy();
});
});
describe("duplex body reassembler — spill file safety", () => {
it("creates the spill directory with mode 0700 and each spill file with mode 0600", async () => {
const receiver = await makeReceiver({ spillThresholdBytes: 4, rawChunkBytes: RAW });
const dirStat = await fs.stat(receiver.spillDir);
expect(dirStat.mode & 0o777).toBe(0o700);
// The final destroy rejects this in-flight body, so attach the catch at once.
const pending = receiver.begin("mode", 8);
pending.catch(() => undefined);
receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "mode", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") });
await new Promise((resolve) => setTimeout(resolve, 20));
const entries = await fs.readdir(receiver.spillDir);
expect(entries).toHaveLength(1);
// The file name is random, so no frame id rides the path.
expect(entries[0]).not.toContain("mode");
const fileStat = await fs.stat(path.join(receiver.spillDir, entries[0]));
expect(fileStat.mode & 0o777).toBe(0o600);
await receiver.destroy();
});
});
describe("duplex body reassembler — startup sweep", () => {
it("removes a stale spill directory from a dead process but keeps a live one", async () => {
const deadPid = 999999; // A pid that is almost never alive.
const staleDir = path.join(spillRoot, `paperclip-duplex-spill-${deadPid}-stale`);
await fs.mkdir(staleDir);
await fs.writeFile(path.join(staleDir, "leftover.body"), "x");
const liveDir = path.join(spillRoot, `paperclip-duplex-spill-${process.pid}-live`);
await fs.mkdir(liveDir);
await sweepStaleSpillDirectories(spillRoot);
await expect(fs.stat(staleDir)).rejects.toThrow();
// The sweep keeps a directory that belongs to a live process.
await expect(fs.stat(liveDir)).resolves.toBeDefined();
});
});
@@ -0,0 +1,596 @@
/**
* Receive-side body reassembly and spill for the sandbox duplex channel.
*
* The duplex codec splits one request body or one response body into an envelope
* frame plus a sequence of `body_chunk` frames. This module reassembles those
* chunks back into one body. The receiver reads `bodyByteCount` from the envelope
* first, then chooses one of two paths before it reads the first chunk:
* - memory path: a body at or below the spill threshold stays in a memory
* buffer.
* - spill path: a body above the spill threshold streams to a temporary file,
* so the receiver never holds a whole large body in memory.
*
* The spill path bounds the disk a single channel can consume. A per-channel
* arena caps the bytes on disk and the number of open spill files. The receiver
* reserves a file slot and reserves bytes before it opens a file and before each
* write. A reserve that exceeds a cap fails closed: it rejects the body as a
* resource error and unlinks any partial file.
*
* The reassembler treats every frame as untrusted. It rejects a bad chunk as a
* loud, terminal protocol error: a non-canonical base64 payload, a wrong chunk
* size, an out-of-order `seq`, a total that does not equal `bodyByteCount`, or a
* chunk that arrives for a completed or unknown body.
*/
import { randomBytes } from "node:crypto";
import { createReadStream } from "node:fs";
import { promises as fs } from "node:fs";
import type { FileHandle } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { Readable } from "node:stream";
import {
DUPLEX_BODY_CHUNK_RAW_BYTES,
type DuplexBodyChunkFrame,
} from "./duplex-frame-codec.js";
/**
* The default spill threshold, in bytes. A body at or below this size stays in
* memory. A body above it spills to disk. The value is 1 MiB. It stays
* injectable so a test can lower it and exercise the spill path.
*/
export const DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES = 1024 * 1024;
/**
* The default per-channel spill byte cap. The receiver keeps at most this many
* bytes on disk at one time for one channel. The value is 64 MiB. It is about
* 0.64% of the default 10 GiB sandbox disk and stays far below the security
* ceiling of 25% of the disk quota. It stays injectable so a test can lower it.
*/
export const DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP = 64 * 1024 * 1024;
/**
* The default per-channel open-file cap. The receiver keeps at most this many
* spill files open at one time for one channel. The value derives from the byte
* cap and the spill threshold: 64 MiB divided by the 1 MiB spill floor gives 64.
* It stays injectable so a test can lower it.
*/
export const DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP = 64;
/** The prefix of every spill directory name. The startup sweep matches it. */
const SPILL_DIR_PREFIX = "paperclip-duplex-spill-";
/** The config a caller passes to build a {@link DuplexBodyReceiver}. */
export interface DuplexBodyReceiverConfig {
/** The spill threshold in bytes. Defaults to {@link DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES}. */
spillThresholdBytes?: number;
/** The fixed raw chunk size in bytes. Defaults to {@link DUPLEX_BODY_CHUNK_RAW_BYTES}. */
rawChunkBytes?: number;
/** The per-channel spill byte cap. Defaults to {@link DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP}. */
channelSpillByteCap?: number;
/** The per-channel open-file cap. Defaults to {@link DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP}. */
channelSpillFileCap?: number;
/** The root directory that holds the spill directory. Defaults to the OS temp dir. */
spillRootDir?: string;
/** A diagnostic sink for a non-fatal cleanup note. The receiver names no body id here. */
logger?: (message: string) => void;
}
/** The reason the reassembler rejected a body. Each reason is a terminal protocol error. */
export type DuplexBodyErrorCode =
| "chunk_without_envelope"
| "chunk_after_completion"
| "seq_out_of_order"
| "chunk_not_canonical_base64"
| "chunk_wrong_size"
| "empty_chunk"
| "body_overrun"
| "zero_body_has_chunk"
| "spill_cap_exceeded"
| "spill_write_failed"
| "path_escape"
| "channel_closed";
/** A reassembly error. The receiver returns it; it never leaks a raw path or a raw provider string. */
export class DuplexBodyError extends Error {
readonly code: DuplexBodyErrorCode;
constructor(code: DuplexBodyErrorCode, message: string) {
super(message);
this.name = "DuplexBodyError";
this.code = code;
}
}
/**
* One reassembled body. The memory path holds the whole body in a buffer. The
* spill path holds a temporary file. A caller reads the memory buffer, or opens
* a fresh read stream, then disposes the body once.
*/
export interface ReassembledBody {
/** The raw byte length of the body. */
readonly byteCount: number;
/** True when the body spilled to disk. */
readonly spilled: boolean;
/**
* The whole body as a buffer. The memory path returns it. The spill path
* throws, because the spill path must never load a whole large body into
* memory on the receive side. A caller streams a spilled body instead.
*/
readMemory(): Buffer;
/** A fresh read stream over the body. The memory path streams one buffer; the spill path streams the file. */
createReadStream(): Readable;
/** Release the body. The spill path unlinks the file and releases the reservation exactly once. */
dispose(): Promise<void>;
}
/**
* The per-channel arena that bounds the spill disk use. It tracks the bytes on
* disk and the open files, and it owns the spill directory. It hands out
* reservations and takes them back. The reservation math is synchronous, so a
* concurrent reserve never races past a cap.
*/
class SpillArena {
usedBytes = 0;
openFiles = 0;
constructor(
readonly dir: string,
private readonly byteCap: number,
private readonly fileCap: number,
) {}
/** Reserve one open-file slot. Return false when the file cap is already at the limit. */
reserveFile(): boolean {
if (this.openFiles >= this.fileCap) return false;
this.openFiles += 1;
return true;
}
/** Release one open-file slot. */
releaseFile(): void {
if (this.openFiles > 0) this.openFiles -= 1;
}
/** Reserve `n` bytes of disk. Return false when the byte cap would be exceeded. */
reserveBytes(n: number): boolean {
if (this.usedBytes + n > this.byteCap) return false;
this.usedBytes += n;
return true;
}
/** Release `n` bytes of disk. */
releaseBytes(n: number): void {
this.usedBytes -= n;
if (this.usedBytes < 0) this.usedBytes = 0;
}
}
/** The internal state of one in-flight body. */
interface ActiveBody {
bodyByteCount: number;
nextSeq: number;
receivedBytes: number;
spilled: boolean;
/** Memory-path accumulation. Empty for the spill path. */
memoryChunks: Buffer[];
/** The spill file path. Null for the memory path. */
filePath: string | null;
/** The open spill file handle. Null for the memory path or before the first write. */
handle: FileHandle | null;
/** The bytes this body reserved in the arena. The dispose path releases exactly this many. */
reservedBytes: number;
/** True when the body holds an arena file slot that the dispose path must release. */
holdsFileSlot: boolean;
/** True once the dispose path ran, so it runs its release once. */
disposed: boolean;
/** Resolve the completion promise the caller awaits. */
resolve: (body: ReassembledBody) => void;
/** Reject the completion promise on a terminal error. */
reject: (error: DuplexBodyError) => void;
/** Serialize the spill writes, so the chunks land on disk in `seq` order. */
writeChain: Promise<void>;
/** True once the body settled (resolved or rejected), so a late chunk never double-settles. */
settled: boolean;
}
/** The synchronous result of one `pushChunk` call. */
export type DuplexBodyPushResult =
| { ok: true }
| { ok: false; error: DuplexBodyError };
/**
* A per-channel receive-side body reassembler. One instance serves one channel.
* The caller feeds it one envelope per body, then the `body_chunk` frames for
* that body. The reassembler reassembles the body on the memory path or the
* spill path and settles one completion promise per body.
*/
export class DuplexBodyReceiver {
private readonly spillThresholdBytes: number;
private readonly rawChunkBytes: number;
private readonly arena: SpillArena;
private readonly logger?: (message: string) => void;
private readonly active = new Map<string, ActiveBody>();
private constructor(arena: SpillArena, config: DuplexBodyReceiverConfig) {
this.arena = arena;
this.spillThresholdBytes =
config.spillThresholdBytes ?? DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES;
this.rawChunkBytes = config.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES;
this.logger = config.logger;
}
/**
* Build a receiver. The receiver owns the spill directory. It creates the
* directory with `mkdtemp` and mode 0700, then sweeps stale spill directories
* from a prior process. The sweep runs and awaits before the receiver serves a
* body, so no caller races a half-swept directory.
*/
static async create(config: DuplexBodyReceiverConfig = {}): Promise<DuplexBodyReceiver> {
const root = config.spillRootDir ?? os.tmpdir();
await sweepStaleSpillDirectories(root, config.logger);
const dir = await fs.mkdtemp(path.join(root, `${SPILL_DIR_PREFIX}${process.pid}-`));
// `mkdtemp` creates the directory with mode 0700 on a POSIX host by default,
// but a permissive umask can widen it. Set the mode so the owner alone reads,
// writes, and lists the spill directory.
await fs.chmod(dir, 0o700);
const arena = new SpillArena(
dir,
config.channelSpillByteCap ?? DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP,
config.channelSpillFileCap ?? DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP,
);
return new DuplexBodyReceiver(arena, config);
}
/** The spill directory this receiver owns. A test reads it to assert containment and cleanup. */
get spillDir(): string {
return this.arena.dir;
}
/** The bytes on disk across all in-flight bodies for this channel. A test asserts the reservation. */
get spillUsedBytes(): number {
return this.arena.usedBytes;
}
/** The open spill files for this channel. A test asserts the file-slot reservation. */
get spillOpenFiles(): number {
return this.arena.openFiles;
}
/**
* Begin one body. The caller passes the envelope `id` and `bodyByteCount`. The
* receiver chooses the memory path or the spill path from `bodyByteCount`
* before the first chunk arrives. A `bodyByteCount` of 0 completes at once with
* an empty body and accepts no chunk. The returned promise resolves with the
* reassembled body, or rejects with a {@link DuplexBodyError} on a terminal
* protocol or resource error.
*/
begin(id: string, bodyByteCount: number): Promise<ReassembledBody> {
return new Promise<ReassembledBody>((resolve, reject) => {
// A repeated envelope id for an in-flight body is a protocol violation. The
// caller must not begin the same id twice. Reject the new begin and leave
// the in-flight body untouched.
if (this.active.has(id)) {
reject(new DuplexBodyError("chunk_after_completion", "duplicate body envelope id"));
return;
}
const body: ActiveBody = {
bodyByteCount,
nextSeq: 0,
receivedBytes: 0,
spilled: bodyByteCount > this.spillThresholdBytes,
memoryChunks: [],
filePath: null,
handle: null,
reservedBytes: 0,
holdsFileSlot: false,
disposed: false,
resolve,
reject,
writeChain: Promise.resolve(),
settled: false,
};
this.active.set(id, body);
if (bodyByteCount === 0) {
// The zero-length body completes at once. It accepts no chunk, so the
// receiver keeps the id in a completed state to reject a stray chunk.
this.complete(id, body);
return;
}
if (body.spilled) {
// Reserve one file slot before the receiver opens the spill file. A
// reserve past the file cap fails closed at once.
if (!this.arena.reserveFile()) {
this.settleReject(id, body, new DuplexBodyError("spill_cap_exceeded", "spill file cap reached"));
return;
}
body.holdsFileSlot = true;
// Open the spill file with a random name and mode 0600. The name never
// derives from the frame id, so no id rides a path. Verify the resolved
// path stays inside the spill directory before the receiver writes.
this.openSpillFile(id, body);
}
});
}
/**
* Push one `body_chunk` frame. The receiver validates the chunk against the
* in-flight body, then appends it on the memory path or the spill path. The
* return value reports a synchronous validation outcome. A spill write failure
* or a completion rejects the begin promise instead.
*/
pushChunk(frame: DuplexBodyChunkFrame): DuplexBodyPushResult {
const body = this.active.get(frame.id);
if (!body) {
return fail("chunk_without_envelope", "body_chunk arrived with no matching envelope");
}
if (body.bodyByteCount === 0) {
return fail("zero_body_has_chunk", "a zero-length body accepts no body_chunk");
}
if (body.settled) {
return fail("chunk_after_completion", "body_chunk arrived after the body completed");
}
if (frame.seq !== body.nextSeq) {
return fail("seq_out_of_order", "body_chunk seq is a duplicate, a gap, or a reorder");
}
// Decode the base64 payload and confirm it is canonical. A non-canonical
// payload re-encodes to a different string, so the round-trip catches it.
const decoded = Buffer.from(frame.data, "base64");
if (decoded.toString("base64") !== frame.data) {
return fail("chunk_not_canonical_base64", "body_chunk data is not canonical base64");
}
if (decoded.length === 0) {
return fail("empty_chunk", "body_chunk is empty");
}
const nextReceived = body.receivedBytes + decoded.length;
if (nextReceived > body.bodyByteCount) {
return fail("body_overrun", "body_chunk overruns the declared body size");
}
const isFinal = nextReceived === body.bodyByteCount;
if (!isFinal && decoded.length !== this.rawChunkBytes) {
// A non-final chunk must be exactly the fixed raw slice size. A short
// non-final chunk is a truncation; an over-size chunk is a framing fault.
return fail("chunk_wrong_size", "a non-final body_chunk is not the fixed slice size");
}
if (decoded.length > this.rawChunkBytes) {
// The final chunk must also stay within the fixed raw slice size.
return fail("chunk_wrong_size", "a body_chunk exceeds the fixed slice size");
}
body.nextSeq += 1;
body.receivedBytes = nextReceived;
if (body.spilled) {
this.appendSpill(frame.id, body, decoded, isFinal);
} else {
body.memoryChunks.push(decoded);
if (isFinal) this.complete(frame.id, body);
}
return { ok: true };
}
/**
* Dispose one body by id, whatever its state. The dispose path unlinks the
* spill file and releases the arena reservation exactly once. The caller runs
* it on every settle path and on channel death, so a spill file never lingers.
*/
async disposeBody(id: string): Promise<void> {
const body = this.active.get(id);
if (!body) return;
this.active.delete(id);
await this.release(body);
}
/**
* Dispose every in-flight body. The caller runs it on channel death, so a
* channel that dies mid-body leaves no spill file behind.
*/
async disposeAll(): Promise<void> {
const ids = [...this.active.keys()];
await Promise.all(ids.map((id) => this.disposeBody(id)));
}
/**
* Remove the spill directory this receiver owns. The caller runs it after it
* disposes every body, on channel shutdown.
*/
async destroy(): Promise<void> {
await this.disposeAll();
await fs.rm(this.arena.dir, { recursive: true, force: true }).catch(() => undefined);
}
private openSpillFile(id: string, body: ActiveBody): void {
const name = `${randomBytes(24).toString("hex")}.body`;
const filePath = path.join(this.arena.dir, name);
// Path containment. The resolved file path must stay inside the spill
// directory. A generated random name never escapes, but the check is defense
// in depth against a future name scheme.
const resolvedDir = path.resolve(this.arena.dir);
const resolvedFile = path.resolve(filePath);
if (resolvedFile !== path.join(resolvedDir, name) || !resolvedFile.startsWith(resolvedDir + path.sep)) {
this.settleReject(id, body, new DuplexBodyError("path_escape", "spill path escapes the spill directory"));
return;
}
body.filePath = filePath;
// Open the file once, exclusive, mode 0600. The write chain appends each
// chunk in order to this one handle.
body.writeChain = body.writeChain.then(async () => {
body.handle = await fs.open(filePath, "wx", 0o600);
});
body.writeChain.catch((error) => {
this.settleReject(id, body, new DuplexBodyError("spill_write_failed", errorText(error)));
});
}
private appendSpill(id: string, body: ActiveBody, decoded: Buffer, isFinal: boolean): void {
// Reserve the bytes before the write. A reserve past the byte cap fails
// closed: reject the body and unlink the partial file.
if (!this.arena.reserveBytes(decoded.length)) {
this.settleReject(id, body, new DuplexBodyError("spill_cap_exceeded", "spill byte cap reached"));
return;
}
body.reservedBytes += decoded.length;
body.writeChain = body.writeChain.then(async () => {
if (body.settled) return;
if (!body.handle) throw new Error("spill file handle is not open");
await body.handle.write(decoded);
if (isFinal) {
await body.handle.close();
body.handle = null;
this.complete(id, body);
}
});
body.writeChain.catch((error) => {
if (error instanceof DuplexBodyError) return;
this.settleReject(id, body, new DuplexBodyError("spill_write_failed", errorText(error)));
});
}
private complete(id: string, body: ActiveBody): void {
if (body.settled) return;
body.settled = true;
const receiver = this;
const handle: ReassembledBody = {
byteCount: body.bodyByteCount,
spilled: body.spilled,
readMemory(): Buffer {
if (body.spilled) {
throw new DuplexBodyError("spill_write_failed", "a spilled body has no in-memory buffer");
}
return Buffer.concat(body.memoryChunks);
},
createReadStream(): Readable {
if (body.spilled && body.filePath) return createReadStream(body.filePath);
return Readable.from(Buffer.concat(body.memoryChunks));
},
async dispose(): Promise<void> {
await receiver.disposeBody(id);
},
};
body.resolve(handle);
}
private settleReject(id: string, body: ActiveBody, error: DuplexBodyError): void {
if (body.settled) return;
body.settled = true;
// Release the reservation and unlink the partial file before the reject, so
// a rejected body never keeps a spill file or a reservation. The reject fires
// after the release, so a caller that awaits the reject sees a clean arena.
void this.disposeBody(id).finally(() => body.reject(error));
}
private async release(body: ActiveBody): Promise<void> {
if (body.disposed) return;
body.disposed = true;
// Wait for the pending writes to settle, so the unlink never races an open
// handle. Swallow a write error here; the reject path already reported it.
await body.writeChain.catch(() => undefined);
if (body.handle) {
await body.handle.close().catch(() => undefined);
body.handle = null;
}
if (body.filePath) {
await fs.rm(body.filePath, { force: true }).catch((error) => {
this.logger?.(`Duplex spill unlink failed: ${errorText(error)}`);
});
}
if (body.reservedBytes > 0) {
this.arena.releaseBytes(body.reservedBytes);
body.reservedBytes = 0;
}
if (body.holdsFileSlot) {
this.arena.releaseFile();
body.holdsFileSlot = false;
}
// A body that the channel disposes before it settles must not leave its
// begin promise pending. Reject it with a terminal channel-closed error, so
// a caller that awaits the body sees the channel death. A body that already
// settled keeps its outcome: `complete` resolves it, and `settleReject` sets
// `settled` before it calls dispose, so this guard never double-settles.
if (!body.settled) {
body.settled = true;
body.reject(
new DuplexBodyError("channel_closed", "the channel closed before the body completed"),
);
}
}
}
function fail(code: DuplexBodyErrorCode, message: string): DuplexBodyPushResult {
return { ok: false, error: new DuplexBodyError(code, message) };
}
function errorText(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
/**
* Split a whole body buffer into `body_chunk` frames. The send side reads the
* source body once, then calls this to build the frames. Each frame carries one
* raw slice of `rawChunkBytes` as base64 text, except the final frame, which
* carries the remaining bytes. A zero-length body yields no frame.
*/
export function splitBodyIntoChunkFrames(
id: string,
body: Buffer,
version: number,
rawChunkBytes: number = DUPLEX_BODY_CHUNK_RAW_BYTES,
): DuplexBodyChunkFrame[] {
const frames: DuplexBodyChunkFrame[] = [];
let seq = 0;
for (let offset = 0; offset < body.length; offset += rawChunkBytes) {
const slice = body.subarray(offset, offset + rawChunkBytes);
frames.push({
version,
type: "body_chunk",
id,
seq,
data: slice.toString("base64"),
});
seq += 1;
}
return frames;
}
/**
* Sweep stale spill directories from a prior process. The receiver runs it at
* startup, before it serves a body. It removes a spill directory whose embedded
* pid is not this process and is not a live process, so a crash that left a
* directory behind does not leak disk. It never removes a directory of a live
* process, so a concurrent channel keeps its own spill directory.
*/
export async function sweepStaleSpillDirectories(
root: string,
logger?: (message: string) => void,
): Promise<void> {
const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []);
await Promise.all(
entries.map(async (entry) => {
if (!entry.isDirectory() || !entry.name.startsWith(SPILL_DIR_PREFIX)) return;
const pid = parsePidFromSpillDir(entry.name);
if (pid !== null && pid !== process.pid && isProcessAlive(pid)) return;
if (pid === process.pid) return;
await fs.rm(path.join(root, entry.name), { recursive: true, force: true }).catch((error) => {
logger?.(`Duplex stale spill sweep failed: ${errorText(error)}`);
});
}),
);
}
/** Read the pid segment from a spill directory name, or null when it is absent. */
function parsePidFromSpillDir(name: string): number | null {
const rest = name.slice(SPILL_DIR_PREFIX.length);
const dash = rest.indexOf("-");
const pidText = dash === -1 ? rest : rest.slice(0, dash);
const pid = Number(pidText);
return Number.isInteger(pid) && pid > 0 ? pid : null;
}
/** Return true when a process with the pid is alive. A signal 0 probes without a real signal. */
function isProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch (error) {
// EPERM means the process exists but this process cannot signal it. ESRCH
// means no such process. Treat only ESRCH as dead.
return (error as NodeJS.ErrnoException).code === "EPERM";
}
}
@@ -1,317 +0,0 @@
import { afterEach, describe, expect, it } from "vitest";
import {
createDuplexBridgeBroker,
type DuplexBridgeBroker,
type DuplexBrokerForwardResult,
} from "./duplex-bridge-broker.js";
import {
DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
DuplexAggregateByteLedger,
} from "./duplex-aggregate-byte-ledger.js";
import {
DUPLEX_FRAME_VERSION,
DuplexFrameDecoder,
encodeDuplexFrame,
type DuplexRequestFrame,
type DuplexResponseFrame,
} from "./duplex-frame-codec.js";
import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
/**
* Regression harness for the broker aggregate byte ledger charging.
*
* The harness feeds request frames straight into the broker through an in-memory
* channel, the same as a provider that controls the transport. A test controls
* when each forward settles, so it can assert the ledger charge while the forward
* is still in flight and after it settles.
*/
/** One pending forward a test settles by hand. */
interface PendingForward {
id: string;
resolve: (result: DuplexBrokerForwardResult) => void;
reject: (error: Error) => void;
settled: boolean;
}
/** The in-memory channel plus the levers a test uses to drive the broker. */
interface FakeChannelHarness {
channel: CommandManagedDuplexChannel;
feed: (frame: DuplexRequestFrame) => void;
exit: () => void;
responses: DuplexResponseFrame[];
forwards: PendingForward[];
resolveForward: (id: string, body?: string) => void;
}
/** Build one valid request frame. */
function requestFrame(id: string, method = "POST"): DuplexRequestFrame {
return {
version: DUPLEX_FRAME_VERSION,
type: "request",
id,
method,
path: `/api/issues/${id}`,
query: "",
headers: { "content-type": "application/json" },
body: JSON.stringify({ id }),
};
}
function createFakeChannelHarness(): FakeChannelHarness {
const responses: DuplexResponseFrame[] = [];
const forwards: PendingForward[] = [];
const writtenDecoder = new DuplexFrameDecoder();
let dataListener: ((chunk: string) => void) | null = null;
let exitListener: ((exit: { exitCode: number | null }) => void) | null = null;
const channel: CommandManagedDuplexChannel = {
write: (data) => {
for (const result of writtenDecoder.push(data)) {
if (result.ok && result.frame.type === "response") responses.push(result.frame);
}
},
onData: (listener) => {
dataListener = listener;
},
onExit: (listener) => {
exitListener = listener;
},
stop: () => undefined,
close: () => Promise.resolve(),
};
return {
channel,
feed: (frame) => {
if (!dataListener) throw new Error("The broker did not bind the data listener.");
dataListener(encodeDuplexFrame(frame));
},
exit: () => {
if (!exitListener) throw new Error("The broker did not bind the exit listener.");
exitListener({ exitCode: 0 });
},
responses,
forwards,
resolveForward: (id, body = JSON.stringify({ ok: true })) => {
const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled);
if (!forward) throw new Error(`No unsettled forward for id ${id}.`);
forward.settled = true;
forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body });
},
};
}
/** A forward handler that hands each call to the harness and never auto-resolves. */
function controllableForward(harness: FakeChannelHarness) {
return (request: DuplexRequestFrame): Promise<DuplexBrokerForwardResult> =>
new Promise<DuplexBrokerForwardResult>((resolve, reject) => {
harness.forwards.push({ id: request.id, resolve, reject, settled: false });
});
}
/** Wait for the microtask queue to drain, so a settled promise runs its handlers. */
async function flush(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
}
describe("duplex bridge broker aggregate byte ledger", () => {
const brokers: DuplexBridgeBroker[] = [];
afterEach(async () => {
while (brokers.length > 0) {
const broker = brokers.pop();
if (broker) await broker.close();
}
});
it("charges request-frame, request-payload, and seen-id tokens, then releases the request tokens on forward settlement", async () => {
const harness = createFakeChannelHarness();
const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
duplexAggregateByteLedger: ledger,
});
brokers.push(broker);
broker.start();
harness.feed(requestFrame("req-1"));
// The dispatch retains three tokens: the raw frame, the normalized payload,
// and the no-replay set entry.
expect(ledger.liveTokenCount).toBe(3);
expect(ledger.bytesInUse).toBeGreaterThan(0);
const chargedInFlight = ledger.bytesInUse;
harness.resolveForward("req-1");
await flush();
// The forward settled. The finally owner released the request-frame and the
// request-payload tokens. The seen-id token stays charged for the channel
// lifetime, so exactly one token remains.
expect(ledger.liveTokenCount).toBe(1);
expect(ledger.bytesInUse).toBeGreaterThan(0);
expect(ledger.bytesInUse).toBeLessThan(chargedInFlight);
// A close releases the seen-id token, so the ledger returns to zero.
await broker.close();
expect(ledger.bytesInUse).toBe(0);
expect(ledger.liveTokenCount).toBe(0);
});
it("keeps request tokens charged when the response timer answers before the forward settles, and releases them on settlement", async () => {
const harness = createFakeChannelHarness();
const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
// Squeeze the nested budgets so the response timer fires quickly while the
// forward stays unsettled.
budgets: { forwardTimeoutMs: 5, responseBudgetMs: 10, gatewayWaitMs: 20 },
duplexAggregateByteLedger: ledger,
});
brokers.push(broker);
broker.start();
harness.feed(requestFrame("req-1", "GET"));
expect(ledger.liveTokenCount).toBe(3);
// Wait past the response budget so the backstop answers the gateway while the
// forward is still in flight.
await new Promise((resolve) => setTimeout(resolve, 40));
expect(harness.responses.some((frame) => frame.id === "req-1")).toBe(true);
// The gateway got its answer, but the forward has not settled. The request
// tokens and the seen-id token stay charged: nothing released yet.
expect(ledger.liveTokenCount).toBe(3);
// Settle the orphaned forward. Its finally owner now releases the two request
// tokens; the seen-id token still stays for the channel lifetime.
harness.resolveForward("req-1");
await flush();
expect(ledger.liveTokenCount).toBe(1);
await broker.close();
expect(ledger.bytesInUse).toBe(0);
});
it("refuses a one-byte-over dispatch with the fixed marker and retains nothing", async () => {
const harness = createFakeChannelHarness();
// Measure the exact per-request charge with a throwaway broker, so the test
// can size a ceiling one byte below it and force the reservation to fail.
const probeHarness = createFakeChannelHarness();
const measured = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
const measuringBroker = createDuplexBridgeBroker({
channel: probeHarness.channel,
forwardRequest: controllableForward(probeHarness),
duplexAggregateByteLedger: measured,
});
measuringBroker.start();
probeHarness.feed(requestFrame("req-1"));
const perRequestBytes = measured.bytesInUse;
await measuringBroker.close();
const ledger = new DuplexAggregateByteLedger({ ceilingBytes: perRequestBytes - 1 });
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
duplexAggregateByteLedger: ledger,
});
brokers.push(broker);
broker.start();
harness.feed(requestFrame("req-1"));
await flush();
// The broker retained nothing: no token, no forward, no seen id.
expect(ledger.liveTokenCount).toBe(0);
expect(ledger.bytesInUse).toBe(0);
expect(harness.forwards.length).toBe(0);
// The refusal carries the fixed marker and is a bounded terminal response.
const refusal = harness.responses.find((frame) => frame.id === "req-1");
expect(refusal).toBeTruthy();
expect(refusal?.status).toBe(503);
expect(refusal?.body).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED);
// The refusal did not retain the id, so a resend after pressure eases is
// admitted: raise the ceiling and re-feed.
const roomyLedger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
const roomyHarness = createFakeChannelHarness();
const roomyBroker = createDuplexBridgeBroker({
channel: roomyHarness.channel,
forwardRequest: controllableForward(roomyHarness),
duplexAggregateByteLedger: roomyLedger,
});
brokers.push(roomyBroker);
roomyBroker.start();
roomyHarness.feed(requestFrame("req-1"));
expect(roomyHarness.forwards.length).toBe(1);
});
it("releases every retained token on a terminal channel loss", async () => {
const harness = createFakeChannelHarness();
const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
duplexAggregateByteLedger: ledger,
});
brokers.push(broker);
broker.start();
harness.feed(requestFrame("req-1"));
harness.feed(requestFrame("req-2"));
expect(ledger.liveTokenCount).toBe(6);
// The channel exits mid-flight. The broker latches the loss, transfers the
// in-flight forwards to the orphan registry, and releases the seen-id tokens.
harness.exit();
expect(broker.runDisposition.failed).toBe(true);
// The seen-id tokens released at loss; the two request tokens per forward stay
// charged until each aborted forward settles.
expect(ledger.liveTokenCount).toBe(4);
// The aborted forwards settle. Each finally owner releases its two request
// tokens, so the ledger returns to zero.
harness.forwards[0]?.reject(new Error("aborted"));
harness.forwards[1]?.reject(new Error("aborted"));
await flush();
expect(ledger.bytesInUse).toBe(0);
expect(ledger.liveTokenCount).toBe(0);
});
it("does not double-release a token when a forward settles after the channel closed", async () => {
const harness = createFakeChannelHarness();
let underflow = 0;
const ledger = new DuplexAggregateByteLedger({
ceilingBytes: 1_000_000,
telemetry: {
setBytesInUse() {},
recordReservationRejection() {},
recordAccountingUnderflow() {
underflow += 1;
},
},
});
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
duplexAggregateByteLedger: ledger,
});
brokers.push(broker);
broker.start();
harness.feed(requestFrame("req-1"));
await broker.close();
// The close aborted the in-flight forward and orphaned it. The forward now
// settles after the close: its finally owner releases the request tokens once.
harness.forwards[0]?.reject(new Error("aborted"));
await flush();
expect(ledger.bytesInUse).toBe(0);
expect(ledger.liveTokenCount).toBe(0);
// No accounting defect: every token released exactly one time.
expect(underflow).toBe(0);
});
});
@@ -1,946 +0,0 @@
import { afterEach, describe, expect, it } from "vitest";
import {
assertDuplexBrokerLimits,
createDuplexBridgeBroker,
type DuplexBridgeBroker,
type DuplexBrokerForwardResult,
} from "./duplex-bridge-broker.js";
import {
DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES,
DUPLEX_FRAME_VERSION,
DuplexFrameDecoder,
encodeDuplexFrame,
type DuplexRequestFrame,
type DuplexResponseFrame,
} from "./duplex-frame-codec.js";
import {
createDuplexTelemetry,
DUPLEX_COUNTER_LOSS_TOTAL,
DUPLEX_SPAN_REQUEST,
type DuplexTelemetryCounterRecord,
type DuplexTelemetryEventRecord,
type DuplexTelemetrySpanRecord,
} from "./duplex-telemetry.js";
import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
/**
* Unit harness for the host broker limit gate.
*
* The harness drives frames straight into the broker through an in-memory
* channel. It does not spawn the gateway, so a test injects untrusted request
* frames directly, the same as a malicious provider that controls the transport.
* The channel records each frame the broker writes back, so a test asserts the
* exact response the broker returns for a refused request.
*/
/** One pending forward the test controls. */
interface PendingForward {
id: string;
resolve: (result: DuplexBrokerForwardResult) => void;
reject: (error: Error) => void;
settled: boolean;
}
/** The in-memory channel plus the levers a test uses to drive the broker. */
interface FakeChannelHarness {
channel: CommandManagedDuplexChannel;
/** Push one request frame into the broker read path. */
feed: (frame: DuplexRequestFrame) => void;
/** The response frames the broker wrote back, in order. */
responses: DuplexResponseFrame[];
/** Every forward call the broker made, in order. */
forwards: PendingForward[];
/** Resolve the newest unsettled forward for one id with a 200 result. */
resolveForward: (id: string, body?: string) => void;
/** Resolve every unsettled forward with a 200 result. */
resolveAll: () => void;
}
/** Build one valid request frame with distinctive, secret-looking fields. */
function requestFrame(id: string, method = "POST"): DuplexRequestFrame {
return {
version: DUPLEX_FRAME_VERSION,
type: "request",
id,
method,
path: `/api/issues/${id}`,
query: "?secret-query=leak",
headers: { authorization: "Bearer super-secret-provider-token" },
body: JSON.stringify({ secret: "secret-request-body" }),
};
}
/**
* Build the in-memory channel harness. The channel keeps the broker read
* listener, so `feed` pushes an encoded frame into the broker. The channel
* decodes each written frame, so `responses` holds the response frames only.
*/
function createFakeChannelHarness(): FakeChannelHarness {
const responses: DuplexResponseFrame[] = [];
const forwards: PendingForward[] = [];
const writtenDecoder = new DuplexFrameDecoder();
let dataListener: ((chunk: string) => void) | null = null;
const channel: CommandManagedDuplexChannel = {
write: (data) => {
for (const result of writtenDecoder.push(data)) {
if (result.ok && result.frame.type === "response") {
responses.push(result.frame);
}
}
},
onData: (listener) => {
dataListener = listener;
},
onExit: () => undefined,
stop: () => undefined,
close: () => Promise.resolve(),
};
return {
channel,
feed: (frame) => {
if (!dataListener) throw new Error("The broker did not bind the data listener.");
dataListener(encodeDuplexFrame(frame));
},
responses,
forwards,
resolveForward: (id, body = JSON.stringify({ ok: true })) => {
const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled);
if (!forward) throw new Error(`No unsettled forward for id ${id}.`);
forward.settled = true;
forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body });
},
resolveAll: () => {
for (const forward of forwards) {
if (forward.settled) continue;
forward.settled = true;
forward.resolve({
status: 200,
headers: { "content-type": "application/json" },
body: JSON.stringify({ ok: true }),
});
}
},
};
}
/** A forward handler that hands each call to the harness and never auto-resolves. */
function controllableForward(harness: FakeChannelHarness) {
return (request: DuplexRequestFrame): Promise<DuplexBrokerForwardResult> =>
new Promise<DuplexBrokerForwardResult>((resolve, reject) => {
harness.forwards.push({ id: request.id, resolve, reject, settled: false });
});
}
/** The telemetry sink capture. It proves the broker records nothing for a refusal. */
interface TelemetryCapture {
spans: DuplexTelemetrySpanRecord[];
counters: DuplexTelemetryCounterRecord[];
events: DuplexTelemetryEventRecord[];
}
/** Build the real telemetry facade over a capturing recorder. */
function createTelemetryCapture(): { telemetry: ReturnType<typeof createDuplexTelemetry>; capture: TelemetryCapture } {
const capture: TelemetryCapture = { spans: [], counters: [], events: [] };
const telemetry = createDuplexTelemetry({
providerKey: "daytona",
recorder: {
recordSpan: (record) => capture.spans.push(record),
incrementCounter: (record) => capture.counters.push(record),
emitEvent: (record) => capture.events.push(record),
},
});
return { telemetry, capture };
}
describe("duplex bridge broker request limits", () => {
const brokers: DuplexBridgeBroker[] = [];
afterEach(async () => {
while (brokers.length > 0) {
const broker = brokers.pop();
if (broker) await broker.close();
}
});
it("rejects a non-positive or non-integer limit at construction", () => {
expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 0, maxLifetimeRequests: 10 })).toThrow();
expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 5, maxLifetimeRequests: 0 })).toThrow();
expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 1.5, maxLifetimeRequests: 10 })).toThrow();
expect(() =>
assertDuplexBrokerLimits({ maxInFlightRequests: Number.POSITIVE_INFINITY, maxLifetimeRequests: 10 }),
).toThrow();
expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 8, maxLifetimeRequests: 16 })).not.toThrow();
});
it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", () => {
const harness = createFakeChannelHarness();
const { telemetry, capture } = createTelemetryCapture();
const maxInFlightRequests = 4;
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
telemetry,
maxInFlightRequests,
});
brokers.push(broker);
broker.start();
// Inject more than the limit of unique valid frames. Every forward hangs, so
// the broker holds the maximum number of in-flight forwards.
const total = 10;
const ids = Array.from({ length: total }, (_unused, index) => `flight-${index}`);
for (const id of ids) harness.feed(requestFrame(id));
// The broker forwarded only up to the limit. It refused the rest.
expect(harness.forwards).toHaveLength(maxInFlightRequests);
expect(harness.responses).toHaveLength(total - maxInFlightRequests);
for (const response of harness.responses) {
expect(response.status).toBe(503);
expect(response.outcome).toBe("unavailable");
expect(response.headers["x-paperclip-bridge-outcome"]).toBe("unavailable");
expect(JSON.parse(response.body)).toEqual({
error: "Duplex broker capacity limit reached.",
outcome: "unavailable",
retryable: true,
});
}
// The refusal leaks no route, query, body, or token.
const refusalText = harness.responses.map((response) => encodeDuplexFrame(response)).join("");
expect(refusalText).not.toContain("secret");
expect(refusalText).not.toContain("super-secret-provider-token");
expect(refusalText).not.toContain("/api/issues/");
// The broker recorded no request span and no counter for a refusal. Only the
// delivered requests below produce a span.
expect(capture.spans).toHaveLength(0);
expect(capture.counters).toHaveLength(0);
// Resolve the in-flight forwards. The broker delivers one response for each
// and drains the pending bookkeeping.
harness.resolveAll();
});
it("frees capacity after a delivered request and forwards a resent refused id (no-replay preserved)", async () => {
const harness = createFakeChannelHarness();
const maxInFlightRequests = 2;
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
maxInFlightRequests,
});
brokers.push(broker);
broker.start();
// Saturate the in-flight limit with two hung forwards.
harness.feed(requestFrame("keep-0"));
harness.feed(requestFrame("keep-1"));
expect(harness.forwards).toHaveLength(2);
// A third unique id is refused, retryable. The broker did not forward it and
// did not retain its id.
harness.feed(requestFrame("resend-me"));
expect(harness.forwards).toHaveLength(2);
expect(harness.responses).toHaveLength(1);
expect(JSON.parse(harness.responses[0].body).retryable).toBe(true);
// Free one in-flight slot. The delivered request drains the pending record.
harness.resolveForward("keep-0");
await Promise.resolve();
// The gateway resends the refused id. Capacity is free now, so the broker
// forwards it exactly one time. This proves the refusal did not poison the id.
harness.feed(requestFrame("resend-me"));
expect(harness.forwards.filter((forward) => forward.id === "resend-me")).toHaveLength(1);
// A resend of an already-forwarded id never reaches the forward twice.
const forwardedKeep1 = harness.forwards.filter((forward) => forward.id === "keep-1").length;
harness.feed(requestFrame("keep-1"));
expect(harness.forwards.filter((forward) => forward.id === "keep-1")).toHaveLength(forwardedKeep1);
harness.resolveAll();
});
it("bounds the retained request ids over the channel lifetime and refuses each new id past the limit", async () => {
const harness = createFakeChannelHarness();
const { telemetry, capture } = createTelemetryCapture();
const maxLifetimeRequests = 3;
const broker = createDuplexBridgeBroker({
channel: harness.channel,
// Resolve every forward at once, so each dispatched id leaves the in-flight
// count but stays in the retained-id set.
forwardRequest: async (request: DuplexRequestFrame): Promise<DuplexBrokerForwardResult> => {
harness.forwards.push({ id: request.id, resolve: () => undefined, reject: () => undefined, settled: true });
return { status: 200, headers: { "content-type": "application/json" }, body: JSON.stringify({ ok: true }) };
},
telemetry,
maxLifetimeRequests,
});
brokers.push(broker);
broker.start();
// Inject more than the lifetime limit of unique valid frames.
const total = 9;
for (let index = 0; index < total; index += 1) {
harness.feed(requestFrame(`life-${index}`));
await Promise.resolve();
}
// The broker forwarded only up to the lifetime limit. The retained-id set
// stays bounded, because a refused id never joins it.
expect(harness.forwards).toHaveLength(maxLifetimeRequests);
// The broker delivered a real response for each forwarded request, then a
// bounded terminal refusal for each id past the limit.
const refusals = harness.responses.filter((response) => response.status === 503);
expect(refusals).toHaveLength(total - maxLifetimeRequests);
for (const refusal of refusals) {
expect(refusal.outcome).toBe("unavailable");
// A lifetime refusal is terminal, so a resend does not help.
expect(JSON.parse(refusal.body).retryable).toBe(false);
}
// A resend of a refused id stays refused, so the set never grows.
harness.feed(requestFrame("life-8"));
await Promise.resolve();
expect(harness.forwards).toHaveLength(maxLifetimeRequests);
// The telemetry surface stayed on the fixed request span for the delivered
// requests only. No refusal produced a span, a counter, or a loss record.
expect(capture.spans).toHaveLength(maxLifetimeRequests);
for (const span of capture.spans) {
expect(span.name).toBe(DUPLEX_SPAN_REQUEST);
expect(span.dimensions.outcome).toBe("ok");
expect(Object.keys(span.dimensions).sort()).toEqual(["outcome", "provider", "transport"]);
}
expect(capture.counters).toHaveLength(0);
expect(broker.lossRecord).toBeNull();
expect(broker.state).toBe("open");
});
it("forwards nothing and fails the channel closed under a flood of over-limit ids", () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
});
brokers.push(broker);
broker.start();
// Send a sustained flood of distinct ids, each one byte over the id bound. The
// codec rejects each frame on the read path, so no over-limit id ever reaches
// the retained-id set or a forward.
const overLimitId = (index: number): string =>
`${"a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES)}-${index}`;
for (let index = 0; index < 20; index += 1) {
harness.feed(requestFrame(overLimitId(index)));
}
// The broker forwarded nothing. The retained-id set never grew, because the
// set only grows on a dispatched forward, and the broker dispatched none.
expect(harness.forwards).toHaveLength(0);
// The first over-limit frame is a protocol failure, so the broker fails the
// whole channel closed. It stays lost for the rest of the flood.
expect(broker.state).toBe("lost");
expect(broker.lossRecord?.reason).toBe("protocol_failure");
});
it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
});
brokers.push(broker);
broker.start();
// An id at the maximum byte size is allowed. The broker forwards it one time.
const maxId = "a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES);
harness.feed(requestFrame(maxId));
expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1);
// Deliver the response, then resend the same id. The broker retained the id,
// so the resend never reaches the forward a second time.
harness.resolveForward(maxId);
harness.feed(requestFrame(maxId));
expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1);
expect(broker.state).toBe("open");
});
it("maps a forward that resolves with the indeterminate marker to a non-retryable response and retains the id", async () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
});
brokers.push(broker);
broker.start();
// The forward handler resolves with a 504 that carries the indeterminate
// marker. This is the shape the host forward returns for a response-read
// failure after the host commit. The broker must keep the marker and the
// non-retryable body, so the gateway maps it to a non-retryable status and a
// caller never repeats a possibly-committed mutation.
harness.feed(requestFrame("commit-1"));
const forward = harness.forwards.find((entry) => entry.id === "commit-1" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
forward.settled = true;
forward.resolve({
status: 504,
headers: {
"content-type": "application/json",
"x-paperclip-bridge-outcome": "indeterminate",
},
body: JSON.stringify({
error: "Bridge response body exceeded the configured size limit of 32 bytes.",
outcome: "indeterminate",
retryable: false,
}),
});
// The broker answers inside the forward `then` microtask, so let it settle.
await Promise.resolve();
expect(harness.responses).toHaveLength(1);
const response = harness.responses[0]!;
expect(response.status).toBe(504);
expect(response.outcome).toBe("indeterminate");
expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate");
expect(JSON.parse(response.body)).toEqual({
error: "Bridge response body exceeded the configured size limit of 32 bytes.",
outcome: "indeterminate",
retryable: false,
});
// The broker delivered a response, so it retained the id. A resend never
// reaches the forward a second time, so a caller that ignores the
// non-retryable status still cannot repeat the mutation through the broker.
harness.feed(requestFrame("commit-1"));
expect(harness.forwards.filter((entry) => entry.id === "commit-1")).toHaveLength(1);
expect(broker.state).toBe("open");
});
it("maps a forward rejection for a mutating method to a non-retryable indeterminate response and retains the id", async () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
});
brokers.push(broker);
broker.start();
// The forward rejects before the host delivers a response. A fetch can
// reject after the request reaches the host and the host commits, but
// before the response headers arrive. The broker did not abort the
// forward, so a POST must return a non-retryable indeterminate response.
// A retryable status would let a caller repeat a committed mutation.
harness.feed(requestFrame("mutate-1", "POST"));
const forward = harness.forwards.find((entry) => entry.id === "mutate-1" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
forward.settled = true;
forward.reject(new Error("fetch failed before the response headers arrived"));
// The broker answers inside the forward rejection microtask, so let it settle.
await new Promise((resolve) => setTimeout(resolve, 0));
expect(harness.responses).toHaveLength(1);
const response = harness.responses[0]!;
expect(response.status).toBe(504);
expect(response.outcome).toBe("indeterminate");
expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate");
expect(JSON.parse(response.body)).toEqual({
error: "fetch failed before the response headers arrived",
outcome: "indeterminate",
retryable: false,
});
// The broker delivered a response, so it retained the id. A resend never
// reaches the forward a second time, so a caller that ignores the
// non-retryable status still cannot repeat the mutation through the broker.
harness.feed(requestFrame("mutate-1", "POST"));
expect(harness.forwards.filter((entry) => entry.id === "mutate-1")).toHaveLength(1);
expect(broker.state).toBe("open");
});
it("substitutes a bounded terminal response for an oversized response and keeps the channel open", async () => {
const harness = createFakeChannelHarness();
const { telemetry, capture } = createTelemetryCapture();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
telemetry,
// A small frame bound makes a legitimately large response body exceed the
// bound, so the test exercises the encode guard without a megabyte body.
maxFrameBytes: 1000,
});
brokers.push(broker);
broker.start();
// Two requests are in flight at the same time. One resolves with a normal
// body; the other resolves with a body that pushes the response frame over
// the bound. The host produced both results, so both requests reached the
// host.
harness.feed(requestFrame("small-1", "POST"));
harness.feed(requestFrame("big-1", "POST"));
const smallForward = harness.forwards.find((entry) => entry.id === "small-1" && !entry.settled);
if (!smallForward) throw new Error("The broker did not forward the small request.");
smallForward.settled = true;
smallForward.resolve({
status: 200,
headers: { "content-type": "application/json" },
body: JSON.stringify({ ok: true }),
});
await new Promise((resolve) => setTimeout(resolve, 0));
const bigForward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled);
if (!bigForward) throw new Error("The broker did not forward the big request.");
bigForward.settled = true;
bigForward.resolve({
status: 200,
headers: { "content-type": "application/json" },
body: JSON.stringify({ data: "y".repeat(2000) }),
});
await new Promise((resolve) => setTimeout(resolve, 0));
// The other in-flight request delivered its full response, unaffected by the
// oversized one.
expect(harness.responses).toHaveLength(2);
const delivered = harness.responses.find((entry) => entry.id === "small-1")!;
expect(delivered.status).toBe(200);
expect(delivered.outcome).toBe("completed");
expect(JSON.parse(delivered.body)).toEqual({ ok: true });
// The oversized response became one bounded terminal response marked
// indeterminate and non-retryable, so a caller does not retry a possible
// mutation. The broker did not write the oversized frame.
const terminal = harness.responses.find((entry) => entry.id === "big-1")!;
expect(terminal.status).toBe(502);
expect(terminal.outcome).toBe("indeterminate");
expect(terminal.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate");
expect(JSON.parse(terminal.body)).toEqual({
error: "upstream response too large to deliver",
outcome: "indeterminate",
retryable: false,
});
// The oversized body never crossed the wire. The broker did not truncate it or
// pass it through.
const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join("");
expect(writtenText).not.toContain("yyyy");
// The broker recorded the oversized request as an error outcome, never a loss.
// The channel stays open and the run does not fail.
const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error");
expect(errorSpans).toHaveLength(1);
expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST);
expect(broker.lossRecord).toBeNull();
expect(broker.state).toBe("open");
expect(broker.runDisposition.failed).toBe(false);
});
it("sends a minimal terminal response when the frame bound rejects the full replacement", async () => {
const harness = createFakeChannelHarness();
const { telemetry, capture } = createTelemetryCapture();
// The bound accepts a compact request (130 bytes) and the minimal terminal
// response (106 bytes), but rejects the full replacement response (288 bytes).
// This proves the broker never drops the channel or strands the request when
// the configured bound is smaller than the full replacement response.
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
telemetry,
maxFrameBytes: 200,
});
brokers.push(broker);
broker.start();
// A compact request frame fits the bound, so the broker forwards it.
harness.feed({
version: DUPLEX_FRAME_VERSION,
type: "request",
id: "big-1",
method: "POST",
path: "/api/issues/big-1",
query: "",
headers: {},
body: JSON.stringify({ go: 1 }),
});
const forward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
forward.settled = true;
// The host produced a large response body that pushes the real response frame
// and the full replacement frame over the bound.
forward.resolve({
status: 200,
headers: { "content-type": "application/json" },
body: JSON.stringify({ data: "y".repeat(2000) }),
});
await new Promise((resolve) => setTimeout(resolve, 0));
// The broker sent one minimal terminal response, not the oversized real
// response and not the full replacement. The minimal response carries the
// indeterminate outcome, so the gateway maps it to a terminal 409.
expect(harness.responses).toHaveLength(1);
const terminal = harness.responses[0]!;
expect(terminal.id).toBe("big-1");
expect(terminal.status).toBe(502);
expect(terminal.outcome).toBe("indeterminate");
expect(terminal.headers).toEqual({});
expect(terminal.body).toBe("");
// The oversized body never crossed the wire.
const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join("");
expect(writtenText).not.toContain("yyyy");
// The broker recorded the request as an error outcome, never a loss. The
// channel stays open and the run does not fail.
const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error");
expect(errorSpans).toHaveLength(1);
expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST);
expect(broker.lossRecord).toBeNull();
expect(broker.state).toBe("open");
expect(broker.runDisposition.failed).toBe(false);
});
it("keeps the channel open and logs when the bound rejects even the minimal terminal response", async () => {
const harness = createFakeChannelHarness();
const { telemetry, capture } = createTelemetryCapture();
const logs: string[] = [];
// The bound accepts a tiny request (99 bytes) but rejects even the minimal
// terminal response (102 bytes). The broker cannot send any frame for this
// request, so it must keep the channel open and log a clear local error.
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
telemetry,
maxFrameBytes: 100,
logger: (message) => logs.push(message),
});
brokers.push(broker);
broker.start();
harness.feed({
version: DUPLEX_FRAME_VERSION,
type: "request",
id: "x",
method: "GET",
path: "/",
query: "",
headers: {},
body: "",
});
const forward = harness.forwards.find((entry) => entry.id === "x" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
forward.settled = true;
forward.resolve({
status: 200,
headers: { "content-type": "application/json" },
body: JSON.stringify({ data: "y".repeat(200) }),
});
await new Promise((resolve) => setTimeout(resolve, 0));
// The broker sent no response frame, because no frame fits the bound.
expect(harness.responses).toHaveLength(0);
// The broker recorded the request as an error outcome, never a loss.
const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error");
expect(errorSpans).toHaveLength(1);
// The broker logged a clear local error and kept the channel open.
expect(logs.some((line) => line.includes("could not deliver a terminal response"))).toBe(true);
expect(broker.lossRecord).toBeNull();
expect(broker.state).toBe("open");
expect(broker.runDisposition.failed).toBe(false);
});
it("completes a request within the bound and delivers the real response", async () => {
const harness = createFakeChannelHarness();
const { telemetry, capture } = createTelemetryCapture();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
telemetry,
maxFrameBytes: 1000,
});
brokers.push(broker);
broker.start();
harness.feed(requestFrame("ok-1", "POST"));
harness.resolveForward("ok-1", JSON.stringify({ ok: true }));
await new Promise((resolve) => setTimeout(resolve, 0));
// The broker delivered the real response promptly with the completed outcome.
expect(harness.responses).toHaveLength(1);
const delivered = harness.responses[0]!;
expect(delivered.id).toBe("ok-1");
expect(delivered.status).toBe(200);
expect(delivered.outcome).toBe("completed");
expect(JSON.parse(delivered.body)).toEqual({ ok: true });
// The delivered response records an `ok` span, never an error, and never a loss.
const okSpans = capture.spans.filter((span) => span.dimensions.outcome === "ok");
expect(okSpans).toHaveLength(1);
const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error");
expect(errorSpans).toHaveLength(0);
expect(broker.lossRecord).toBeNull();
expect(broker.state).toBe("open");
expect(broker.runDisposition.failed).toBe(false);
});
it("maps a forward rejection for a safe method to a retryable response", async () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
});
brokers.push(broker);
broker.start();
// A safe method never changes host state, so a retry cannot double-apply a
// mutation. A forward rejection for a GET stays retryable: the broker
// returns a 502 with the completed outcome, so the gateway passes it through.
harness.feed(requestFrame("read-1", "GET"));
const forward = harness.forwards.find((entry) => entry.id === "read-1" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
forward.settled = true;
forward.reject(new Error("fetch failed before the response headers arrived"));
await new Promise((resolve) => setTimeout(resolve, 0));
expect(harness.responses).toHaveLength(1);
const response = harness.responses[0]!;
expect(response.status).toBe(502);
expect(response.outcome).toBe("completed");
expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined();
expect(JSON.parse(response.body)).toEqual({
error: "fetch failed before the response headers arrived",
});
});
it("maps a forward-budget timeout for a safe method to a retryable response", async () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
// A tiny forward budget aborts the controller before the manual rejection.
budgets: { forwardTimeoutMs: 5 },
});
brokers.push(broker);
broker.start();
// A safe method never changes host state. A forward timeout for a GET must
// stay retryable: the broker returns a 504 with the completed outcome and no
// indeterminate marker, so the gateway does not map it to a terminal 409.
harness.feed(requestFrame("read-timeout-1", "GET"));
const forward = harness.forwards.find((entry) => entry.id === "read-timeout-1" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
// Wait for the forward-budget timer to abort the controller, then reject the
// forward. The rejection handler now sees the aborted signal.
await new Promise((resolve) => setTimeout(resolve, 20));
forward.settled = true;
forward.reject(new Error("Duplex broker forward budget exceeded."));
await new Promise((resolve) => setTimeout(resolve, 0));
expect(harness.responses).toHaveLength(1);
const response = harness.responses[0]!;
expect(response.status).toBe(504);
expect(response.outcome).toBe("completed");
expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined();
expect(JSON.parse(response.body)).toEqual({
error: "Duplex broker forward budget exceeded.",
retryable: true,
});
});
it("maps a forward-budget timeout for a mutating method to a non-retryable indeterminate response", async () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
budgets: { forwardTimeoutMs: 5 },
});
brokers.push(broker);
broker.start();
// A POST may commit before the forward budget aborts the call. The broker
// must keep the terminal contract: a 504 with the indeterminate outcome, so
// the gateway maps it to a non-retryable 409 and no caller double-applies it.
harness.feed(requestFrame("mutate-timeout-1", "POST"));
const forward = harness.forwards.find((entry) => entry.id === "mutate-timeout-1" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
await new Promise((resolve) => setTimeout(resolve, 20));
forward.settled = true;
forward.reject(new Error("Duplex broker forward budget exceeded."));
await new Promise((resolve) => setTimeout(resolve, 0));
expect(harness.responses).toHaveLength(1);
const response = harness.responses[0]!;
expect(response.status).toBe(504);
expect(response.outcome).toBe("indeterminate");
expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate");
expect(JSON.parse(response.body)).toEqual({
error: "Duplex broker forward budget exceeded.",
outcome: "indeterminate",
retryable: false,
});
});
it("maps a response-budget backstop for a safe method to a retryable response", async () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
// The forward budget aborts the call, but the forward promise stays
// pending. A GET receives the response headers, but the body reader stalls
// through the response budget, so the forward promise never settles. The
// response-budget backstop must answer the request.
budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 },
});
brokers.push(broker);
broker.start();
// A safe method never changes host state, so a stalled body reader cannot
// leave a mutation half-applied. The backstop must keep the request
// retryable: a 504 with the completed outcome and no indeterminate marker,
// so the gateway does not map it to a terminal 409.
harness.feed(requestFrame("read-stall-1", "GET"));
const forward = harness.forwards.find((entry) => entry.id === "read-stall-1" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
// Never settle the forward. Wait past the response budget so the backstop
// fires on the stalled body reader.
await new Promise((resolve) => setTimeout(resolve, 30));
expect(harness.responses).toHaveLength(1);
const response = harness.responses[0]!;
expect(response.status).toBe(504);
expect(response.outcome).toBe("completed");
expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined();
expect(JSON.parse(response.body)).toEqual({
error: "Duplex broker response budget exceeded.",
retryable: true,
});
});
it("maps a response-budget backstop for a mutating method to a non-retryable indeterminate response", async () => {
const harness = createFakeChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: controllableForward(harness),
budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 },
});
brokers.push(broker);
broker.start();
// A POST may commit before the response budget passes, and the broker cannot
// prove the host applied no mutation. The backstop must keep the terminal
// contract: a 504 with the indeterminate outcome, so the gateway maps it to a
// non-retryable 409 and no caller double-applies it.
harness.feed(requestFrame("mutate-stall-1", "POST"));
const forward = harness.forwards.find((entry) => entry.id === "mutate-stall-1" && !entry.settled);
if (!forward) throw new Error("The broker did not forward the request.");
await new Promise((resolve) => setTimeout(resolve, 30));
expect(harness.responses).toHaveLength(1);
const response = harness.responses[0]!;
expect(response.status).toBe(504);
expect(response.outcome).toBe("indeterminate");
expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate");
expect(JSON.parse(response.body)).toEqual({
error: "Duplex broker response budget exceeded.",
outcome: "indeterminate",
retryable: false,
});
});
});
/**
* A channel harness that captures the broker exit listener. `emitExit` invokes it,
* so a test drives one channel exit into the broker.
*/
function createExitChannelHarness(): {
channel: CommandManagedDuplexChannel;
emitExit: (exit: { exitCode: number | null; transportClosed?: boolean }) => void;
} {
let exitListener:
| ((exit: { exitCode: number | null; transportClosed?: boolean }) => void)
| null = null;
const channel: CommandManagedDuplexChannel = {
write: () => undefined,
onData: () => undefined,
onExit: (listener) => {
exitListener = listener;
},
stop: () => undefined,
close: () => Promise.resolve(),
};
return {
channel,
emitExit: (exit) => {
if (!exitListener) throw new Error("The broker did not bind the exit listener.");
exitListener(exit);
},
};
}
describe("duplex bridge broker exit taxonomy", () => {
const brokers: DuplexBridgeBroker[] = [];
afterEach(async () => {
while (brokers.length > 0) {
const broker = brokers.pop();
if (broker) await broker.close();
}
});
it("records a transport close as a distinct loss, not a process exit", () => {
const { telemetry, capture } = createTelemetryCapture();
const harness = createExitChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: () =>
Promise.resolve({ status: 200, headers: {}, body: "" }),
telemetry,
});
brokers.push(broker);
broker.start();
// A reason-less transport close carries the discriminator and no exit code.
harness.emitExit({ exitCode: null, transportClosed: true });
expect(broker.lossRecord?.reason).toBe("transport_closed");
expect(broker.runDisposition).toEqual({ failed: true, lossReason: "transport_closed" });
const lossCounter = capture.counters.find(
(record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL,
);
expect(lossCounter?.dimensions.loss_reason).toBe("transport_closed");
});
it("records a numeric exit as a process exit", () => {
const { telemetry, capture } = createTelemetryCapture();
const harness = createExitChannelHarness();
const broker = createDuplexBridgeBroker({
channel: harness.channel,
forwardRequest: () =>
Promise.resolve({ status: 200, headers: {}, body: "" }),
telemetry,
});
brokers.push(broker);
broker.start();
// A numeric exit code is a real process exit, so the broker keeps the existing
// channel_exit -> provider_exit mapping.
harness.emitExit({ exitCode: 0 });
expect(broker.lossRecord?.reason).toBe("channel_exit");
expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" });
const lossCounter = capture.counters.find(
(record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL,
);
expect(lossCounter?.dimensions.loss_reason).toBe("provider_exit");
});
});
File diff suppressed because it is too large. Load diff
@@ -1,646 +0,0 @@
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import net from "node:net";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { StringDecoder } from "node:string_decoder";
import { afterEach, describe, expect, it } from "vitest";
import {
authorizeSandboxCallbackBridgeRequestWithRoutes,
getSandboxCallbackBridgeServerSource,
SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE,
} from "./sandbox-callback-bridge.js";
import {
DuplexFrameDecoder,
type DuplexFrame,
type DuplexReadyFrame,
type DuplexRequestFrame,
} from "./duplex-frame-codec.js";
import {
createDuplexBridgeBroker,
type DuplexBridgeBroker,
type DuplexBrokerForwardResult,
} from "./duplex-bridge-broker.js";
import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
/**
* Local real-process end-to-end harness for the composed duplex path.
*
* The harness spawns the real generated gateway with plain `node` from a
* temporary file, attaches the real host broker to the child stdin and stdout,
* and forwards each request to a local fake API server on a real HTTP call. It
* uses no provider credentials.
*
* The child stdio pipes are kernel pipes. They give partial reads, split a
* multi-byte UTF-8 sequence across two chunks, apply backpressure, and give a
* true end of file when the child dies. The harness proves the composed path
* handles each of these real conditions.
*/
/** One recorded call the fake API server received. */
interface FakeApiRequest {
method: string;
url: string;
auth: string | null;
runId: string | null;
body: string;
}
/** The responder the fake API server calls for each request. */
type FakeApiResponder = (req: IncomingMessage, res: ServerResponse, body: string) => void;
/** The handle for one fake API server. */
interface FakeApiServer {
origin: string;
requests: FakeApiRequest[];
setResponder: (responder: FakeApiResponder) => void;
/** The count of sockets the server holds open right now. */
openSocketCount: () => number;
/** True while the server still accepts connections. */
listening: () => boolean;
close: () => Promise<void>;
}
/**
* Start a local fake API server. The forward handler targets it, so one real
* HTTP call proves the composed path end to end. The server tracks each open
* socket, so a test asserts teardown leaves no open socket handle.
*/
async function startFakeApiServer(): Promise<FakeApiServer> {
const requests: FakeApiRequest[] = [];
const sockets = new Set<net.Socket>();
let responder: FakeApiResponder = (req, res) => {
const url = new URL(req.url ?? "/", "http://127.0.0.1");
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true, path: url.pathname }));
};
const server = createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk: Buffer) => chunks.push(chunk));
req.on("end", () => {
const body = Buffer.concat(chunks).toString("utf8");
requests.push({
method: req.method ?? "GET",
url: req.url ?? "/",
auth: req.headers.authorization ?? null,
runId:
typeof req.headers["x-paperclip-run-id"] === "string"
? req.headers["x-paperclip-run-id"]
: null,
body,
});
responder(req, res, body);
});
});
server.on("connection", (socket) => {
sockets.add(socket);
socket.on("close", () => sockets.delete(socket));
});
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", () => resolve());
});
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("The fake API server did not expose a TCP port.");
}
return {
origin: `http://127.0.0.1:${address.port}`,
requests,
setResponder: (next) => {
responder = next;
},
openSocketCount: () => sockets.size,
listening: () => server.listening,
close: () =>
new Promise<void>((resolve) => {
// Destroy each open socket first. A keep-alive client socket keeps the
// server open, so `server.close` alone could stall. The destroy makes the
// close callback fire and drives the open socket count to zero.
for (const socket of sockets) socket.destroy();
server.close(() => resolve());
}),
};
}
/** The channel view over the spawned child, plus the frames the child sent host-ward. */
interface ChildDuplexChannel {
channel: CommandManagedDuplexChannel;
observedFrames: DuplexFrame[];
stderr: () => string;
}
/**
* Wrap the spawned child as a {@link CommandManagedDuplexChannel}. The broker
* writes to the child stdin, reads the child stdout, and learns of the child
* exit through this channel.
*
* The host end reads a byte stream from the stdout pipe. A pipe read can split
* one multi-byte UTF-8 character across two chunks. The `StringDecoder` holds
* the bytes of an incomplete character until the next chunk, so the broker only
* ever reads whole characters. The channel keeps a second decoder for
* observation only; it lets the harness assert the READY frame and the request
* frames the child produced, and it never feeds the broker.
*/
function attachChildDuplexChannel(child: ChildProcessWithoutNullStreams): ChildDuplexChannel {
const observed = new DuplexFrameDecoder();
const observedFrames: DuplexFrame[] = [];
const stdoutDecoder = new StringDecoder("utf8");
let dataListener: ((chunk: string) => void) | null = null;
let exitListener: ((exit: { exitCode: number | null }) => void) | null = null;
let pendingText = "";
let pendingExit: { exitCode: number | null } | null = null;
let stderrText = "";
child.stdout.on("data", (buffer: Buffer) => {
for (const result of observed.push(buffer)) {
if (result.ok) observedFrames.push(result.frame);
}
const text = stdoutDecoder.write(buffer);
if (text.length === 0) return;
if (dataListener) dataListener(text);
else pendingText += text;
});
child.stderr.on("data", (buffer: Buffer) => {
stderrText += buffer.toString("utf8");
});
child.on("exit", (code) => {
const exit = { exitCode: code };
if (exitListener) exitListener(exit);
else pendingExit = exit;
});
// Swallow a stdin EPIPE. The broker can write one more frame while the child
// exits; the write fails and the broker records the loss on its own path.
child.stdin.on("error", () => undefined);
const channel: CommandManagedDuplexChannel = {
write: (data) => {
child.stdin.write(data);
},
onData: (listener) => {
dataListener = listener;
if (pendingText.length > 0) {
const replay = pendingText;
pendingText = "";
listener(replay);
}
},
onExit: (listener) => {
exitListener = listener;
if (pendingExit) {
const exit = pendingExit;
pendingExit = null;
listener(exit);
}
},
stop: () => {
child.kill("SIGKILL");
},
close: () =>
new Promise<void>((resolve) => {
// Close the write side. The child stdin reaches end of file, so the
// gateway sees a real EOF.
child.stdin.end(() => resolve());
}),
};
return { channel, observedFrames, stderr: () => stderrText };
}
/** The forward-handler mode. `proxy` calls the fake API; `hang` blocks until abort. */
type ForwardMode = "proxy" | "hang";
/** The options for one harness. */
interface HarnessOptions {
hostApiToken?: string;
runId?: string;
maxBodyBytes?: number;
lossExitGraceMs?: number;
}
/** The full harness handle for one composed duplex path. */
interface DuplexE2EHarness {
baseUrl: string;
bridgeToken: string;
hostApiToken: string;
runId: string;
nonce: string;
broker: DuplexBridgeBroker;
api: FakeApiServer;
child: ChildProcessWithoutNullStreams;
observedFrames: DuplexFrame[];
forwardedRequests: DuplexRequestFrame[];
setForwardMode: (mode: ForwardMode) => void;
stderr: () => string;
killChild: () => Promise<void>;
waitFor: (predicate: () => boolean, message: string, timeoutMs?: number) => Promise<void>;
teardown: () => Promise<void>;
}
/** Reserve one free loopback port. The host assigns it to the gateway. */
async function reserveLoopbackPort(): Promise<number> {
return new Promise<number>((resolve, reject) => {
const probe = net.createServer();
probe.once("error", reject);
probe.listen(0, "127.0.0.1", () => {
const address = probe.address();
if (!address || typeof address === "string") {
probe.close(() => reject(new Error("Could not reserve a loopback port.")));
return;
}
const reserved = address.port;
probe.close(() => resolve(reserved));
});
});
}
/**
* Build the composed duplex path: a spawned gateway child, the real broker on
* the child stdio, and a local fake API server as the forward target.
*/
async function createHarness(options: HarnessOptions = {}): Promise<DuplexE2EHarness> {
const hostApiToken = options.hostApiToken ?? "real-run-jwt";
const runId = options.runId ?? "run-e2e";
const maxBodyBytes = options.maxBodyBytes ?? 1_000_000;
const bridgeToken = "duplex-e2e-bridge-token";
const nonce = "e2e112233445566778899aabbccddeeff";
let forwardMode: ForwardMode = "proxy";
const forwardedRequests: DuplexRequestFrame[] = [];
const api = await startFakeApiServer();
const assignedPort = await reserveLoopbackPort();
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-e2e-"));
const entrypoint = path.join(tmpDir, "gateway.mjs");
await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
const child = spawn(process.execPath, [entrypoint], {
stdio: ["pipe", "pipe", "pipe"],
env: {
...process.env,
PAPERCLIP_API_BRIDGE_MODE: SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE,
PAPERCLIP_BRIDGE_HOST: "127.0.0.1",
PAPERCLIP_BRIDGE_PORT: String(assignedPort),
PAPERCLIP_BRIDGE_NONCE: nonce,
PAPERCLIP_BRIDGE_TOKEN: bridgeToken,
PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes),
...(options.lossExitGraceMs != null
? { PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS: String(options.lossExitGraceMs) }
: {}),
},
}) as ChildProcessWithoutNullStreams;
const { channel, observedFrames, stderr } = attachChildDuplexChannel(child);
const forwardRequest = async (
request: DuplexRequestFrame,
opts: { signal: AbortSignal },
): Promise<DuplexBrokerForwardResult> => {
forwardedRequests.push(request);
// Keep the real route allowlist on the forward seam. The broker forwards
// only an allowed route; it denies any other route with a 403.
const denial = authorizeSandboxCallbackBridgeRequestWithRoutes(request);
if (denial) {
return {
status: 403,
headers: { "content-type": "application/json" },
body: JSON.stringify({ error: denial }),
};
}
if (forwardMode === "hang") {
// Block until the broker aborts the forward. This lets a test hold an
// outstanding request open while it forces a loss.
await new Promise<never>((_resolve, reject) => {
const onAbort = () => reject(new Error("The forward call was aborted."));
if (opts.signal.aborted) {
onAbort();
return;
}
opts.signal.addEventListener("abort", onAbort, { once: true });
});
}
const method = request.method.trim().toUpperCase() || "GET";
const headers = new Headers();
for (const [key, value] of Object.entries(request.headers)) {
if (value.trim().length === 0) continue;
headers.set(key, value);
}
// Apply the real host token and the run id, the same as the file bridge path.
headers.set("authorization", `Bearer ${hostApiToken}`);
headers.set("x-paperclip-run-id", runId);
const target = new URL(`${request.path}${request.query ?? ""}`, api.origin);
const response = await fetch(target, {
method,
headers,
...(method === "GET" || method === "HEAD" ? {} : { body: request.body }),
signal: opts.signal,
});
const body = await response.text();
const outHeaders: Record<string, string> = {};
response.headers.forEach((value, key) => {
if (key.toLowerCase() === "content-length") return;
outHeaders[key] = value;
});
return { status: response.status, headers: outHeaders, body };
};
const broker = createDuplexBridgeBroker({
channel,
forwardRequest,
logger: () => undefined,
});
broker.start();
const waitFor = async (
predicate: () => boolean,
message: string,
timeoutMs = 5000,
): Promise<void> => {
const deadline = Date.now() + timeoutMs;
for (;;) {
if (predicate()) return;
if (Date.now() > deadline) {
throw new Error(`Timed out waiting for ${message}. stderr: ${stderr()}`);
}
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, 20);
timer.unref?.();
});
}
};
const waitForExit = (): Promise<void> =>
new Promise<void>((resolve) => {
if (child.exitCode !== null || child.signalCode !== null) {
resolve();
return;
}
child.once("exit", () => resolve());
});
const killChild = async (): Promise<void> => {
child.kill("SIGKILL");
await waitForExit();
};
let torndown = false;
const teardown = async (): Promise<void> => {
if (torndown) return;
torndown = true;
child.kill("SIGKILL");
await waitForExit();
child.stdin.destroy();
child.stdout.destroy();
child.stderr.destroy();
await api.close();
await rm(tmpDir, { recursive: true, force: true });
};
return {
baseUrl: `http://127.0.0.1:${assignedPort}`,
bridgeToken,
hostApiToken,
runId,
nonce,
broker,
api,
child,
observedFrames,
forwardedRequests,
setForwardMode: (mode) => {
forwardMode = mode;
},
stderr,
killChild,
waitFor,
teardown,
};
}
/**
* Build a large body of multi-byte UTF-8 characters. "€" uses three UTF-8 bytes
* and "😀" uses four. A pipe read boundary at a 65536-byte multiple lands inside
* a "€" sequence, because 65536 is not a multiple of three. This guarantees a
* multi-byte character split across two pipe chunks.
*/
function buildMultiByteBody(targetBytes: number): string {
const marker = "😀-start-😀";
const filler = "€".repeat(Math.ceil(targetBytes / 3));
return `${marker}${filler}${marker}`;
}
describe("duplex bridge local end-to-end harness", () => {
const harnesses: DuplexE2EHarness[] = [];
afterEach(async () => {
while (harnesses.length > 0) {
const harness = harnesses.pop();
if (harness) await harness.teardown();
}
});
const readyPredicate = (harness: DuplexE2EHarness) => () =>
harness.observedFrames.some((frame) => frame.type === "ready");
it("delivers a valid READY frame from the spawned gateway child to the attached broker", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "a READY frame from the gateway child");
const ready = harness.observedFrames.find(
(frame) => frame.type === "ready",
) as DuplexReadyFrame;
expect(ready.version).toBe(1);
expect(ready.nonce).toBe(harness.nonce);
// READY is liveness only. It carries no address data.
expect((ready as unknown as Record<string, unknown>).address).toBeUndefined();
expect((ready as unknown as Record<string, unknown>).port).toBeUndefined();
// The broker read the READY frame and stayed open with no loss.
expect(harness.broker.state).toBe("open");
expect(harness.broker.lossRecord).toBeNull();
}, 20000);
it("carries one real HTTP request through the child and the broker to the fake API unchanged", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
harness.api.setResponder((req, res) => {
const url = new URL(req.url ?? "/", "http://127.0.0.1");
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true, echoedPath: url.pathname }));
});
const response = await fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
expect(response.status).toBe(200);
await expect(response.json()).resolves.toEqual({ ok: true, echoedPath: "/api/agents/me" });
// The fake API saw one request with the real host token and the run id. The
// broker replaced the bridge token, so the fake API never saw it.
expect(harness.api.requests).toHaveLength(1);
expect(harness.api.requests[0]).toMatchObject({
method: "GET",
url: "/api/agents/me",
auth: `Bearer ${harness.hostApiToken}`,
runId: harness.runId,
});
}, 20000);
it("reassembles a large response that spans many pipe chunks", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// The body is larger than the pipe buffer, so the response frame crosses the
// child stdin pipe in many chunks. An ASCII body isolates the chunk-span
// behavior from the multi-byte behavior the next test covers.
const largeBody = "x".repeat(256 * 1024);
expect(Buffer.byteLength(largeBody, "utf8")).toBeGreaterThan(200 * 1024);
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
res.end(largeBody);
});
const response = await fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
expect(response.status).toBe(200);
const received = await response.text();
// The whole body returns complete after it crossed the pipe in many chunks.
expect(received.length).toBe(largeBody.length);
expect(received).toBe(largeBody);
}, 20000);
it("decodes a multi-byte UTF-8 sequence split across chunk borders", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
const multiByteBody = buildMultiByteBody(256 * 1024);
expect(Buffer.byteLength(multiByteBody, "utf8")).toBeGreaterThan(200 * 1024);
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
res.end(multiByteBody);
});
const response = await fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
expect(response.status).toBe(200);
const received = await response.text();
// A multi-byte character split across a pipe chunk border decodes to the
// same character, so the whole body returns unchanged.
expect(received.length).toBe(multiByteBody.length);
expect(received).toBe(multiByteBody);
}, 20000);
it("matches each concurrent request through one child to its own response", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
harness.api.setResponder((req, res) => {
const url = new URL(req.url ?? "/", "http://127.0.0.1");
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ path: url.pathname }));
});
const ids = Array.from({ length: 8 }, (_unused, index) => `issue-${index}`);
const responses = await Promise.all(
ids.map((id) =>
fetch(`${harness.baseUrl}/api/issues/${id}`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
}).then(async (response) => ({ status: response.status, body: await response.json() })),
),
);
responses.forEach((result, index) => {
expect(result.status).toBe(200);
expect(result.body).toEqual({ path: `/api/issues/${ids[index]}` });
});
expect(harness.api.requests).toHaveLength(8);
}, 20000);
it("answers 409 to outstanding and 503 to new requests when the broker closes its write side", async () => {
const harness = await createHarness({ lossExitGraceMs: 3000 });
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Hold the forward open, so the broker sends no response frame. The gateway
// keeps the request outstanding until the loss.
harness.setForwardMode("hang");
const outstanding = fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
void outstanding.catch(() => undefined);
await harness.waitFor(
() => harness.forwardedRequests.length >= 1,
"the broker to receive the forwarded request",
);
// The broker closes its write side, so the child stdin reaches a real EOF.
await harness.broker.close();
const lossResponse = await outstanding;
expect(lossResponse.status).toBe(409);
expect(lossResponse.headers.get("x-paperclip-bridge-outcome")).toBe("indeterminate");
await expect(lossResponse.json()).resolves.toEqual({ error: "outcome_indeterminate" });
const afterLoss = await fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
expect(afterLoss.status).toBe(503);
await expect(afterLoss.json()).resolves.toEqual({ error: "bridge_unavailable" });
}, 20000);
it("moves the broker to lost on a child kill, dispatches nothing more, and leaks no handle after teardown", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Hold one forward open, so a request is in flight when the child dies.
harness.setForwardMode("hang");
const outstanding = fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
void outstanding.catch(() => undefined);
await harness.waitFor(
() => harness.forwardedRequests.length >= 1,
"the broker to receive the forwarded request",
);
const forwardedBeforeKill = harness.forwardedRequests.length;
// A real process kill closes the stdout pipe and ends the child.
await harness.killChild();
await harness.waitFor(
() => harness.broker.state === "lost",
"the broker to record the channel loss",
);
expect(harness.broker.lossRecord?.reason).toBe("channel_exit");
// The broker dispatches nothing more after the loss.
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, 100);
timer.unref?.();
});
expect(harness.forwardedRequests.length).toBe(forwardedBeforeKill);
// The outstanding fetch fails because the child died before it answered.
await expect(outstanding).rejects.toThrow();
await harness.teardown();
// Teardown left no open pipe or socket handle.
expect(harness.child.stdin.destroyed).toBe(true);
expect(harness.child.stdout.destroyed).toBe(true);
expect(harness.child.stderr.destroyed).toBe(true);
expect(harness.api.listening()).toBe(false);
expect(harness.api.openSocketCount()).toBe(0);
}, 20000);
});
@@ -126,7 +126,7 @@ describe("round-trip", () => {
),
),
);
for (const type of ["request", "response", "ready", "heartbeat", "close", "error"]) {
for (const type of ["request", "response", "body_chunk", "ready", "heartbeat", "close", "error"]) {
expect(types).toContain(type);
}
});
@@ -212,8 +212,8 @@ describe("streaming decoder behavior", () => {
method: "POST",
path: "/x",
query: "",
headers: {},
body: "😀",
headers: { "x-note": "😀" },
bodyByteCount: 0,
};
const bytes = Buffer.from(encodeDuplexFrame(frame), "utf8");
// Split inside the four-byte emoji, right before a continuation byte.
@@ -275,7 +275,7 @@ describe("request id byte bound", () => {
path: "/",
query: "",
headers: {},
body: "",
bodyByteCount: 0,
});
}
@@ -314,7 +314,7 @@ describe("request id byte bound", () => {
id,
status: 200,
headers: {},
body: "",
bodyByteCount: 0,
outcome: "completed",
}),
);
@@ -461,8 +461,8 @@ describe("size-checked encode", () => {
type: "response",
id: "big",
status: 200,
headers: {},
body: "x".repeat(2_000),
headers: { "x-pad": "x".repeat(2_000) },
bodyByteCount: 0,
outcome: "completed",
};
expect(() => encodeDuplexFrameChecked(frame, 1_000)).not.toThrow();
@@ -475,22 +475,25 @@ describe("size-checked encode", () => {
// Build a frame whose encoded JSON is exactly N bytes. The guard measures the
// JSON without the newline, so N bytes encodes and N+1 bytes fails. This is the
// same boundary the decoder applies, so encode and decode agree exactly.
const base: DuplexFrame = {
// Pad a header value to grow the encoded JSON by an exact byte count. The
// "x-pad" value adds one byte per character, so the frame lands on an exact
// size the guard measures without the trailing newline.
const pad = (n: number): DuplexFrame => ({
version: DUPLEX_FRAME_VERSION,
type: "response",
id: "boundary",
status: 200,
headers: {},
body: "",
headers: { "x-pad": "x".repeat(n) },
bodyByteCount: 0,
outcome: "completed",
};
const baseBytes = Buffer.byteLength(JSON.stringify(base), "utf8");
});
const baseBytes = Buffer.byteLength(JSON.stringify(pad(0)), "utf8");
const limit = baseBytes + 10;
const atLimit: DuplexFrame = { ...base, body: "x".repeat(10) };
const atLimit = pad(10);
expect(Buffer.byteLength(JSON.stringify(atLimit), "utf8")).toBe(limit);
const okResult = encodeDuplexFrameChecked(atLimit, limit);
expect(okResult.ok).toBe(true);
const overResult = encodeDuplexFrameChecked({ ...base, body: "x".repeat(11) }, limit);
const overResult = encodeDuplexFrameChecked(pad(11), limit);
expect(overResult.ok).toBe(false);
if (!overResult.ok) expect(overResult.error.code).toBe("frame_too_large");
});
@@ -15,6 +15,13 @@
* The decoder never throws on the read path. A malformed, oversized, or
* version-mismatch frame becomes a protocol-error result, not an exception. This
* keeps one bad frame from crashing the read loop.
*
* HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback.
* The `http2_v1` host readiness gate imports {@link decodeDuplexLine} from this
* file to read the one READY line every gateway sends, so this file's READY
* frame path stays live for both transports. `duplex-body-spool.ts` still
* imports the request, response, and body-chunk frame types from this file, so
* this file keeps every frame type here.
*/
import {
@@ -24,7 +31,15 @@ import {
} from "./duplex-aggregate-byte-ledger.js";
/** The wire version this codec reads and writes. */
export const DUPLEX_FRAME_VERSION = 1;
export const DUPLEX_FRAME_VERSION = 2;
/**
* The fixed raw byte size of one body slice. The sender splits a body into
* `body_chunk` frames of this raw size, except the final chunk, which holds the
* remaining bytes. The base64 text of one full slice is about 349 KiB, so each
* encoded `body_chunk` frame stays well under {@link DEFAULT_MAX_DUPLEX_FRAME_BYTES}.
*/
export const DUPLEX_BODY_CHUNK_RAW_BYTES = 256 * 1024;
/**
* The default maximum size of one frame, in bytes. The decoder rejects a longer
@@ -54,6 +69,7 @@ const EMPTY = Buffer.alloc(0);
export const DUPLEX_FRAME_TYPES = {
request: "request",
response: "response",
body_chunk: "body_chunk",
ready: "ready",
heartbeat: "heartbeat",
close: "close",
@@ -63,7 +79,12 @@ export const DUPLEX_FRAME_TYPES = {
/** The outcome of a response frame. A loss response carries a non-completed outcome. */
export type DuplexResponseOutcome = "completed" | "indeterminate" | "unavailable";
/** A request frame. The gateway forwards it to the host API path. */
/**
* A request envelope frame. The gateway forwards it to the host API path. The
* envelope carries `bodyByteCount`, the raw byte length of the request body. The
* body itself rides one or more `body_chunk` frames that share the same `id`. A
* `bodyByteCount` of 0 means an empty body and no `body_chunk` frame follows.
*/
export interface DuplexRequestFrame {
version: number;
type: "request";
@@ -72,20 +93,43 @@ export interface DuplexRequestFrame {
path: string;
query: string;
headers: Record<string, string>;
body: string;
bodyByteCount: number;
}
/** A response frame. The host returns it for one request id. */
/**
* A response envelope frame. The host returns it for one request id. The
* envelope carries `bodyByteCount`, the raw byte length of the response body.
* The body itself rides one or more `body_chunk` frames that share the same
* `id`. A `bodyByteCount` of 0 means an empty body and no `body_chunk` frame
* follows.
*/
export interface DuplexResponseFrame {
version: number;
type: "response";
id: string;
status: number;
headers: Record<string, string>;
body: string;
bodyByteCount: number;
outcome: DuplexResponseOutcome;
}
/**
* A body-chunk frame. One frame carries one raw body slice as base64 text in
* `data`. The frames for one body share the `id` of the envelope. `seq` starts
* at 0 and increases by one for each next chunk, with no gap and no repeat. Each
* raw slice is {@link DUPLEX_BODY_CHUNK_RAW_BYTES} bytes, except the final slice,
* which holds the remaining bytes. The receiver reassembles the slices in `seq`
* order and stops when the total raw byte count equals the envelope
* `bodyByteCount`.
*/
export interface DuplexBodyChunkFrame {
version: number;
type: "body_chunk";
id: string;
seq: number;
data: string;
}
/**
* The READY control frame. The gateway sends it one time after it binds the
* host-assigned listener port. READY is a liveness signal, not an address
@@ -129,6 +173,7 @@ export interface DuplexErrorFrame {
export type DuplexFrame =
| DuplexRequestFrame
| DuplexResponseFrame
| DuplexBodyChunkFrame
| DuplexReadyFrame
| DuplexHeartbeatFrame
| DuplexCloseFrame
@@ -194,6 +239,11 @@ function idWithinLimit(id: string): boolean {
return Buffer.byteLength(id, "utf8") <= DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES;
}
/** Return true when the value is a safe integer that is zero or positive. */
function isSafeNonNegativeInteger(value: unknown): value is number {
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
}
/**
* Encode one frame to a single line of JSON with a trailing newline. `JSON.stringify`
* escapes any newline inside a string value, so the returned line holds no
@@ -261,6 +311,8 @@ function validateFrame(frame: Record<string, unknown>): DuplexDecodeResult {
return validateRequest(frame);
case "response":
return validateResponse(frame);
case "body_chunk":
return validateBodyChunk(frame);
case "ready":
return validateReady(frame);
case "heartbeat":
@@ -280,7 +332,7 @@ function validateRequest(frame: Record<string, unknown>): DuplexDecodeResult {
typeof frame.method !== "string" ||
typeof frame.path !== "string" ||
typeof frame.query !== "string" ||
typeof frame.body !== "string" ||
!isSafeNonNegativeInteger(frame.bodyByteCount) ||
!isStringRecord(frame.headers)
) {
return fail("malformed_frame", "request frame has a missing or wrong-typed field");
@@ -298,7 +350,7 @@ function validateResponse(frame: Record<string, unknown>): DuplexDecodeResult {
if (
typeof frame.id !== "string" ||
typeof frame.status !== "number" ||
typeof frame.body !== "string" ||
!isSafeNonNegativeInteger(frame.bodyByteCount) ||
!isStringRecord(frame.headers) ||
typeof frame.outcome !== "string" ||
!RESPONSE_OUTCOMES.has(frame.outcome)
@@ -313,6 +365,29 @@ function validateResponse(frame: Record<string, unknown>): DuplexDecodeResult {
return ok(frame as unknown as DuplexResponseFrame);
}
/**
* Validate a `body_chunk` frame at the structural level. The codec is stateless,
* so it checks only the per-frame shape here:
* - `id` is a string within {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}.
* - `seq` is a safe integer that is zero or positive.
* - `data` is a string.
* The receive-side reassembler owns the stateful checks: the base64 canonical
* form, the fixed slice size, the `seq` order, and the total against
* `bodyByteCount`. The codec never decodes `data` here.
*/
function validateBodyChunk(frame: Record<string, unknown>): DuplexDecodeResult {
if (typeof frame.id !== "string" || typeof frame.data !== "string") {
return fail("malformed_frame", "body_chunk frame has a missing or wrong-typed field");
}
if (!isSafeNonNegativeInteger(frame.seq)) {
return fail("malformed_frame", "body_chunk frame has a missing or wrong-typed seq");
}
if (!idWithinLimit(frame.id)) {
return fail("id_too_large", "body_chunk frame id exceeds the maximum size");
}
return ok(frame as unknown as DuplexBodyChunkFrame);
}
function validateReady(frame: Record<string, unknown>): DuplexDecodeResult {
// READY carries a liveness nonce, not an address. The schema is strict: a valid
// READY frame holds exactly `version`, `type`, and `nonce`. The decoder rejects
@@ -1,17 +1,17 @@
{
"frameVersion": 1,
"frameVersion": 2,
"defaultMaxFrameBytes": 1000000,
"description": "Shared wire-compatibility vectors for the duplex frame codec. Every codec copy decodes the same bytes. bytes is a UTF-8 byte stream; splitByteOffsets cuts it into chunks; expected lists the ordered decode results (frame or protocol-error code).",
"vectors": [
{
"name": "valid-request",
"category": "valid",
"bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "request",
"id": "req-1",
"method": "GET",
@@ -21,7 +21,30 @@
"accept": "application/json",
"content-type": "application/json"
},
"body": ""
"bodyByteCount": 0
}
}
]
},
{
"name": "valid-request-with-body-count",
"category": "valid",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-body\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":1048576}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 2,
"type": "request",
"id": "req-body",
"method": "GET",
"path": "/api/issues/PAP-1",
"query": "expand=comments",
"headers": {
"accept": "application/json",
"content-type": "application/json"
},
"bodyByteCount": 1048576
}
}
]
@@ -29,19 +52,19 @@
{
"name": "valid-response-completed",
"category": "valid",
"bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n",
"bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":11,\"outcome\":\"completed\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "response",
"id": "req-1",
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"ok\":true}",
"bodyByteCount": 11,
"outcome": "completed"
}
}
@@ -50,19 +73,19 @@
{
"name": "valid-response-indeterminate",
"category": "valid",
"bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-2\",\"status\":409,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"error\\\":\\\"outcome_indeterminate\\\"}\",\"outcome\":\"indeterminate\"}\n",
"bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-2\",\"status\":409,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":33,\"outcome\":\"indeterminate\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "response",
"id": "req-2",
"status": 409,
"headers": {
"content-type": "application/json"
},
"body": "{\"error\":\"outcome_indeterminate\"}",
"bodyByteCount": 33,
"outcome": "indeterminate"
}
}
@@ -71,43 +94,121 @@
{
"name": "valid-response-unavailable",
"category": "valid",
"bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-3\",\"status\":503,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"error\\\":\\\"bridge_unavailable\\\"}\",\"outcome\":\"unavailable\"}\n",
"bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-3\",\"status\":503,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":30,\"outcome\":\"unavailable\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "response",
"id": "req-3",
"status": 503,
"headers": {
"content-type": "application/json"
},
"body": "{\"error\":\"bridge_unavailable\"}",
"bodyByteCount": 30,
"outcome": "unavailable"
}
}
]
},
{
"name": "valid-body-chunk",
"category": "valid",
"bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0,\"data\":\"aGVsbG8gYm9keSBjaHVuaw==\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 2,
"type": "body_chunk",
"id": "req-1",
"seq": 0,
"data": "aGVsbG8gYm9keSBjaHVuaw=="
}
}
]
},
{
"name": "valid-body-chunk-final",
"category": "valid",
"bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":7,\"data\":\"dGFpbA==\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 2,
"type": "body_chunk",
"id": "req-1",
"seq": 7,
"data": "dGFpbA=="
}
}
]
},
{
"name": "valid-ready",
"category": "valid",
"bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\"}\n",
"bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "ready",
"nonce": "9f8e7d6c5b4a39281706f5e4d3c2b1a0"
}
}
]
},
{
"name": "valid-heartbeat",
"category": "valid",
"bytes": "{\"version\":2,\"type\":\"heartbeat\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 2,
"type": "heartbeat"
}
}
]
},
{
"name": "valid-close",
"category": "valid",
"bytes": "{\"version\":2,\"type\":\"close\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 2,
"type": "close"
}
}
]
},
{
"name": "valid-error",
"category": "valid",
"bytes": "{\"version\":2,\"type\":\"error\",\"code\":\"malformed_frame\",\"message\":\"peer reported a malformed frame\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 2,
"type": "error",
"code": "malformed_frame",
"message": "peer reported a malformed frame"
}
}
]
},
{
"name": "invalid-ready-missing-nonce",
"category": "invalid",
"bytes": "{\"version\":1,\"type\":\"ready\"}\n",
"bytes": "{\"version\":2,\"type\":\"ready\"}\n",
"expected": [
{
"error": "malformed_frame"
@@ -117,7 +218,7 @@
{
"name": "invalid-ready-with-address",
"category": "invalid",
"bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"address\":\"http://127.0.0.1:47215\"}\n",
"bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"address\":\"http://127.0.0.1:47215\"}\n",
"expected": [
{
"error": "malformed_frame"
@@ -127,57 +228,13 @@
{
"name": "invalid-ready-with-port",
"category": "invalid",
"bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"port\":47215}\n",
"bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"port\":47215}\n",
"expected": [
{
"error": "malformed_frame"
}
]
},
{
"name": "valid-heartbeat",
"category": "valid",
"bytes": "{\"version\":1,\"type\":\"heartbeat\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"type": "heartbeat"
}
}
]
},
{
"name": "valid-close",
"category": "valid",
"bytes": "{\"version\":1,\"type\":\"close\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"type": "close"
}
}
]
},
{
"name": "valid-error",
"category": "valid",
"bytes": "{\"version\":1,\"type\":\"error\",\"code\":\"malformed_frame\",\"message\":\"peer reported a malformed frame\"}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"type": "error",
"code": "malformed_frame",
"message": "peer reported a malformed frame"
}
}
]
},
{
"name": "invalid-not-json",
"category": "invalid",
@@ -211,7 +268,7 @@
{
"name": "invalid-unknown-type",
"category": "invalid",
"bytes": "{\"version\":1,\"type\":\"teleport\"}\n",
"bytes": "{\"version\":2,\"type\":\"teleport\"}\n",
"expected": [
{
"error": "unknown_type"
@@ -221,7 +278,37 @@
{
"name": "invalid-missing-id",
"category": "invalid",
"bytes": "{\"version\":1,\"type\":\"request\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n",
"bytes": "{\"version\":2,\"type\":\"request\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n",
"expected": [
{
"error": "malformed_frame"
}
]
},
{
"name": "invalid-request-body-count-string",
"category": "invalid",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":\"10\"}\n",
"expected": [
{
"error": "malformed_frame"
}
]
},
{
"name": "invalid-request-body-count-negative",
"category": "invalid",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":-1}\n",
"expected": [
{
"error": "malformed_frame"
}
]
},
{
"name": "invalid-request-body-count-fractional",
"category": "invalid",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":1.5}\n",
"expected": [
{
"error": "malformed_frame"
@@ -231,7 +318,47 @@
{
"name": "invalid-bad-headers",
"category": "invalid",
"bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"accept\":7},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n",
"bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"accept\":7},\"bodyByteCount\":10,\"outcome\":\"completed\"}\n",
"expected": [
{
"error": "malformed_frame"
}
]
},
{
"name": "invalid-body-chunk-negative-seq",
"category": "invalid",
"bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":-1,\"data\":\"eA==\"}\n",
"expected": [
{
"error": "malformed_frame"
}
]
},
{
"name": "invalid-body-chunk-fractional-seq",
"category": "invalid",
"bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":2.5,\"data\":\"eA==\"}\n",
"expected": [
{
"error": "malformed_frame"
}
]
},
{
"name": "invalid-body-chunk-missing-data",
"category": "invalid",
"bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0}\n",
"expected": [
{
"error": "malformed_frame"
}
]
},
{
"name": "invalid-body-chunk-nonstring-data",
"category": "invalid",
"bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0,\"data\":42}\n",
"expected": [
{
"error": "malformed_frame"
@@ -241,14 +368,14 @@
{
"name": "partial-request-split",
"category": "partial",
"bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n",
"splitByteOffsets": [
98
102
],
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "request",
"id": "req-1",
"method": "GET",
@@ -258,7 +385,7 @@
"accept": "application/json",
"content-type": "application/json"
},
"body": ""
"bodyByteCount": 0
}
}
]
@@ -266,23 +393,24 @@
{
"name": "partial-utf8-split",
"category": "partial",
"bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-utf8\",\"method\":\"POST\",\"path\":\"/api/issues/PAP-1/comments\",\"query\":\"\",\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"text\\\":\\\"héllo 😀 wörld\\\"}\"}\n",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-utf8\",\"method\":\"POST\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"content-type\":\"application/json\",\"x-note\":\"héllo 😀 wörld\"},\"bodyByteCount\":0}\n",
"splitByteOffsets": [
177
172
],
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "request",
"id": "req-utf8",
"method": "POST",
"path": "/api/issues/PAP-1/comments",
"query": "",
"path": "/api/issues/PAP-1",
"query": "expand=comments",
"headers": {
"content-type": "application/json"
"content-type": "application/json",
"x-note": "héllo 😀 wörld"
},
"body": "{\"text\":\"héllo 😀 wörld\"}"
"bodyByteCount": 0
}
}
]
@@ -290,16 +418,16 @@
{
"name": "partial-two-frames",
"category": "partial",
"bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":11,\"outcome\":\"completed\"}\n",
"splitByteOffsets": [
7,
199,
343
174,
344
],
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "request",
"id": "req-1",
"method": "GET",
@@ -309,19 +437,19 @@
"accept": "application/json",
"content-type": "application/json"
},
"body": ""
"bodyByteCount": 0
}
},
{
"frame": {
"version": 1,
"version": 2,
"type": "response",
"id": "req-1",
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"ok\":true}",
"bodyByteCount": 11,
"outcome": "completed"
}
}
@@ -330,14 +458,14 @@
{
"name": "partial-no-trailing-newline",
"category": "partial",
"bytes": "{\"version\":1,\"type\":\"heartbeat\"}",
"bytes": "{\"version\":2,\"type\":\"heartbeat\"}",
"expected": []
},
{
"name": "oversized-complete-line",
"category": "oversized",
"maxFrameBytes": 64,
"bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}\n",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"q=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n",
"expected": [
{
"error": "frame_too_large"
@@ -351,23 +479,33 @@
"splitByteOffsets": [
40
],
"bytes": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\n{\"version\":1,\"type\":\"heartbeat\"}\n",
"bytes": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\n{\"version\":2,\"type\":\"heartbeat\"}\n",
"expected": [
{
"error": "frame_too_large"
},
{
"frame": {
"version": 1,
"version": 2,
"type": "heartbeat"
}
}
]
},
{
"name": "version-mismatch-lower",
"category": "versionMismatch",
"bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n",
"expected": [
{
"error": "version_mismatch"
}
]
},
{
"name": "version-mismatch-higher",
"category": "versionMismatch",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n",
"bytes": "{\"version\":3,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n",
"expected": [
{
"error": "version_mismatch"
@@ -387,19 +525,19 @@
{
"name": "request-id-at-max-bytes",
"category": "idBoundValid",
"bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n",
"roundTrip": true,
"expected": [
{
"frame": {
"version": 1,
"version": 2,
"type": "request",
"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"method": "GET",
"path": "/",
"query": "",
"headers": {},
"body": ""
"bodyByteCount": 0
}
}
]
@@ -407,7 +545,17 @@
{
"name": "request-id-over-max-bytes",
"category": "idBoundInvalid",
"bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n",
"bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n",
"expected": [
{
"error": "id_too_large"
}
]
},
{
"name": "body-chunk-id-over-max-bytes",
"category": "idBoundInvalid",
"bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"seq\":0,\"data\":\"eA==\"}\n",
"expected": [
{
"error": "id_too_large"
@@ -421,14 +569,28 @@
"name": "encode-within-bound",
"maxFrameBytes": 200,
"frame": {
"version": 1,
"version": 2,
"type": "request",
"id": "req-enc-1",
"method": "GET",
"path": "/",
"query": "",
"headers": {},
"body": "small"
"bodyByteCount": 5
},
"expected": {
"ok": true
}
},
{
"name": "encode-body-chunk-within-bound",
"maxFrameBytes": 200,
"frame": {
"version": 2,
"type": "body_chunk",
"id": "req-enc-3",
"seq": 0,
"data": "Y2h1bmsgcGF5bG9hZA=="
},
"expected": {
"ok": true
@@ -438,7 +600,7 @@
"name": "encode-control-frame-never-rejected",
"maxFrameBytes": 200,
"frame": {
"version": 1,
"version": 2,
"type": "heartbeat"
},
"expected": {
@@ -449,14 +611,15 @@
"name": "encode-over-bound",
"maxFrameBytes": 200,
"frame": {
"version": 1,
"version": 2,
"type": "response",
"id": "req-enc-2",
"status": 200,
"headers": {
"content-type": "application/json"
"content-type": "application/json",
"x-pad": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
"body": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"bodyByteCount": 0,
"outcome": "completed"
},
"expected": {
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import {
DUPLEX_LOSS_REASONS,
HTTP2_TELEMETRY_EVENT_NAMES,
mapHttp2EventToDuplexLossReason,
normalizeDuplexLossReason,
normalizeDuplexProvider,
type DuplexFallbackReason,
type DuplexTransportValue,
} from "./duplex-observability.js";
describe("duplex observability: HTTP/2 event mapping (accepted security fix 7)", () => {
it("test_http2_events_map_to_the_loss_taxonomy", () => {
// Every closed HTTP/2 event name maps to a value from the existing,
// closed DuplexLossReason set. The map reuses one taxonomy across every
// transport, so no separate HTTP/2-only reason list exists.
for (const event of HTTP2_TELEMETRY_EVENT_NAMES) {
const reason = mapHttp2EventToDuplexLossReason(event);
expect(DUPLEX_LOSS_REASONS).toContain(reason);
}
expect(mapHttp2EventToDuplexLossReason("session_error")).toBe("rpc_failure");
expect(mapHttp2EventToDuplexLossReason("session_goaway")).toBe("transport_closed");
expect(mapHttp2EventToDuplexLossReason("session_stall")).toBe("heartbeat_timeout");
expect(mapHttp2EventToDuplexLossReason("write_error")).toBe("write_error");
expect(mapHttp2EventToDuplexLossReason("transport_closed")).toBe("transport_closed");
expect(mapHttp2EventToDuplexLossReason("channel_exit")).toBe("provider_exit");
});
it("test_an_unknown_telemetry_input_maps_to_other", () => {
// An unknown event name, a raw provider error string, and a missing value
// all map to `other`. No raw text ever reaches a sink through this map.
expect(mapHttp2EventToDuplexLossReason("ECONNRESET: read failed at socket.js:42")).toBe("other");
expect(mapHttp2EventToDuplexLossReason("some_future_event")).toBe("other");
expect(mapHttp2EventToDuplexLossReason(null)).toBe("other");
expect(mapHttp2EventToDuplexLossReason(undefined)).toBe("other");
expect(mapHttp2EventToDuplexLossReason("")).toBe("other");
});
it("keeps the closed HTTP/2 event-name set to the seven named signals", () => {
expect([...HTTP2_TELEMETRY_EVENT_NAMES].sort()).toEqual(
[
"session_error",
"session_goaway",
"session_stall",
"write_error",
"transport_closed",
"channel_exit",
].sort(),
);
});
});
describe("duplex observability: closed transport and fallback-reason values", () => {
it("accepts the http2 transport value and the preface_missing fallback reason", () => {
// A type-level check: these string literals must widen to the exported
// union types with no cast, so a drift in either union breaks the build.
const transport: DuplexTransportValue = "http2";
const fallbackReason: DuplexFallbackReason = "preface_missing";
expect(transport).toBe("http2");
expect(fallbackReason).toBe("preface_missing");
});
});
describe("duplex observability: existing normalization stays intact", () => {
it("still maps an unknown loss cause and an unknown provider key to their closed defaults", () => {
expect(normalizeDuplexLossReason("not_a_real_reason")).toBe("other");
expect(normalizeDuplexProvider("some-unlisted-plugin")).toBe("other");
});
});
@@ -12,7 +12,7 @@
* never reaches a span attribute, a counter label, or an event field.
*
* The module stays free of `@opentelemetry/api` and of the database. The host
* injects a {@link DuplexTelemetryRecorder}; the default is a no-op recorder, so
* injects a {@link DuplexObservabilityRecorder}; the default is a no-op recorder, so
* the whole surface stays inert until the host binds a real recorder. Every
* recorder call sits inside an error swallow, so a telemetry failure never breaks
* the request path.
@@ -59,9 +59,9 @@ export const DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL =
/**
* The closed set of aggregate byte ledger metric names. A test pins this exact
* set, so a new ledger metric name needs an explicit review. Each record uses
* only closed constant dimensions and no dynamic label. The telemetry contract
* documents these metrics under "Aggregate byte ledger metrics" in
* `packages/shared/src/telemetry/README.md`.
* only closed constant dimensions and no dynamic label. The Observability
* contract documents these metrics under "Aggregate byte ledger metrics" in
* `doc/observability.md`.
*/
export const DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES = [
DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE,
@@ -86,8 +86,12 @@ export const DUPLEX_DIMENSION_KEYS = [
/** One dimension key from the closed set. */
export type DuplexDimensionKey = (typeof DUPLEX_DIMENSION_KEYS)[number];
/** The transport a record is about. */
export type DuplexTransportValue = "duplex" | "file";
/**
* The transport a record is about. `duplex` names the retired bespoke frame
* protocol; `http2` names the Node HTTP/2 session over the sandbox channel;
* `file` names the queue-file bridge.
*/
export type DuplexTransportValue = "duplex" | "http2" | "file";
/** The outcome of a record. */
export type DuplexOutcomeValue = "ok" | "error";
@@ -98,9 +102,13 @@ export type DuplexOutcomeValue = "ok" | "error";
* stage: the process-scoped route ceiling was full (`route_busy`), the entrypoint
* sync failed (`entrypoint_sync_failed`), the broker construction failed
* (`broker_construction_failed`), or the channel open failed (`channel_open_failed`).
* The `aggregate_bytes_exceeded` reason names a readiness handshake the host fell
* back because the process aggregate byte ceiling had no room for the readiness
* buffer.
* The `aggregate_bytes_exceeded` reason names a readiness handshake, or an
* `http2` post-preface pre-bind buffer, the host fell back because the
* process aggregate byte ceiling had no room. The `preface_missing` reason
* names a missing or an invalid HTTP/2 client connection preface inside the
* bounded readiness buffer: the host found no valid preface after the
* accepted READY line, aborted the HTTP/2 open, and moved the run to
* `queue_v1` one time.
*/
export type DuplexFallbackReason =
| "gate_off"
@@ -113,7 +121,8 @@ export type DuplexFallbackReason =
| "ready_nonce_mismatch"
| "ready_timeout"
| "contaminated"
| "aggregate_bytes_exceeded";
| "aggregate_bytes_exceeded"
| "preface_missing";
/** The class of a terminal loss, relative to the first request dispatch. */
export type DuplexLossClass = "pre_dispatch" | "post_dispatch";
@@ -155,6 +164,58 @@ export function normalizeDuplexLossReason(value: string | null | undefined): Dup
: "other";
}
/**
* The closed set of HTTP/2 session and stream event names the host maps to a
* loss reason. Each name spells one distinct signal the `http2_v1` transport
* can observe: a session-level protocol fault, a peer GOAWAY, a stalled PING
* watchdog, a rejected host-to-sandbox write, a reason-less transport close,
* or the pseudo-terminal channel process exit.
*/
export const HTTP2_TELEMETRY_EVENT_NAMES = [
"session_error",
"session_goaway",
"session_stall",
"write_error",
"transport_closed",
"channel_exit",
] as const;
/** One event name from the closed HTTP/2 event set. */
export type Http2TelemetryEventName = (typeof HTTP2_TELEMETRY_EVENT_NAMES)[number];
/** The host-owned closed HTTP/2 event-name set. It backs {@link mapHttp2EventToDuplexLossReason}. */
const HTTP2_EVENT_NAMES: ReadonlySet<string> = new Set<string>(HTTP2_TELEMETRY_EVENT_NAMES);
/**
* The map from one closed HTTP/2 event name to the existing, closed
* {@link DuplexLossReason} taxonomy. The map reuses that one taxonomy instead
* of a second, HTTP/2-only reason list, so the closed-set pattern stays one
* set of values across every transport.
*/
const HTTP2_EVENT_TO_LOSS_REASON: Readonly<Record<Http2TelemetryEventName, DuplexLossReason>> = {
session_error: "rpc_failure",
session_goaway: "transport_closed",
session_stall: "heartbeat_timeout",
write_error: "write_error",
transport_closed: "transport_closed",
channel_exit: "provider_exit",
};
/**
* Map one HTTP/2 session or stream event name to the closed
* {@link DuplexLossReason} taxonomy (accepted security fix 7). Return the
* mapped reason when the closed event-name set holds the value. Return
* `other` for any other value or a missing value, so an unknown event name or
* a raw provider string never reaches a sink.
*/
export function mapHttp2EventToDuplexLossReason(
event: string | null | undefined,
): DuplexLossReason {
return typeof event === "string" && HTTP2_EVENT_NAMES.has(event)
? HTTP2_EVENT_TO_LOSS_REASON[event as Http2TelemetryEventName]
: "other";
}
/** The one approved public provider value. */
export const DUPLEX_APPROVED_PROVIDER = "daytona";
/** The constant for any provider key outside the allowlist. */
@@ -182,7 +243,7 @@ export function normalizeDuplexProvider(key: string | null | undefined): DuplexP
* are present only when the record defines them, so a span or a counter never
* carries an empty dimension.
*/
export interface DuplexTelemetryDimensions {
export interface DuplexObservabilityDimensions {
provider: DuplexProviderValue;
transport: DuplexTransportValue;
outcome?: DuplexOutcomeValue;
@@ -192,23 +253,23 @@ export interface DuplexTelemetryDimensions {
}
/** One span record the host records. The request span carries a latency. */
export interface DuplexTelemetrySpanRecord {
export interface DuplexObservabilitySpanRecord {
name: string;
dimensions: DuplexTelemetryDimensions;
dimensions: DuplexObservabilityDimensions;
/** The request latency in milliseconds. Only the request span sets it. */
latencyMs?: number;
}
/** One counter increment the host records. */
export interface DuplexTelemetryCounterRecord {
export interface DuplexObservabilityCounterRecord {
metric: string;
dimensions: DuplexTelemetryDimensions;
dimensions: DuplexObservabilityDimensions;
}
/** One event the host emits. */
export interface DuplexTelemetryEventRecord {
export interface DuplexObservabilityEventRecord {
name: string;
dimensions: DuplexTelemetryDimensions;
dimensions: DuplexObservabilityDimensions;
}
/**
@@ -218,14 +279,14 @@ export interface DuplexTelemetryEventRecord {
* The recorder receives only already-mapped dimensions, so the raw provider key
* never reaches it.
*/
export interface DuplexTelemetryRecorder {
recordSpan(record: DuplexTelemetrySpanRecord): void;
incrementCounter(record: DuplexTelemetryCounterRecord): void;
emitEvent(record: DuplexTelemetryEventRecord): void;
export interface DuplexObservabilityRecorder {
recordSpan(record: DuplexObservabilitySpanRecord): void;
incrementCounter(record: DuplexObservabilityCounterRecord): void;
emitEvent(record: DuplexObservabilityEventRecord): void;
}
/** A no-op recorder. Every method does nothing, so the surface stays inert. */
export const NOOP_DUPLEX_TELEMETRY_RECORDER: DuplexTelemetryRecorder = {
export const NOOP_DUPLEX_OBSERVABILITY_RECORDER: DuplexObservabilityRecorder = {
recordSpan() {},
incrementCounter() {},
emitEvent() {},
@@ -252,7 +313,7 @@ export interface DuplexChannelOpenAttempt {
* provider, so a call site never passes a raw key. It maps each semantic event to
* the fixed names and dimensions, then calls the recorder inside an error swallow.
*/
export interface DuplexTelemetry {
export interface DuplexObservability {
/** Begin a channel-open attempt. The caller reports `ready` or `fallback`. */
startChannelOpen(): DuplexChannelOpenAttempt;
/**
@@ -272,12 +333,19 @@ export interface DuplexTelemetry {
recordSessionLeak(): void;
}
/** The options for {@link createDuplexTelemetry}. */
export interface DuplexTelemetryOptions {
/** The options for {@link createDuplexObservability}. */
export interface DuplexObservabilityOptions {
/** The injected recorder. The default is the no-op recorder. */
recorder?: DuplexTelemetryRecorder | null;
recorder?: DuplexObservabilityRecorder | null;
/** The raw provider key. The facade maps it through the allowlist one time. */
providerKey?: string | null;
/**
* The transport value the facade stamps on every non-file record (a channel
* open, a request, a loss, a session leak). The default is `duplex`, so an
* existing caller that names no transport sees no change. The host passes
* `http2` for the `http2_v1` path.
*/
transport?: DuplexTransportValue;
}
/**
@@ -286,25 +354,26 @@ export interface DuplexTelemetryOptions {
* inside a `try/catch`, so a throwing recorder never breaks the request path. A
* missing recorder yields a facade whose methods do nothing.
*/
export function createDuplexTelemetry(options: DuplexTelemetryOptions = {}): DuplexTelemetry {
const recorder = options.recorder ?? NOOP_DUPLEX_TELEMETRY_RECORDER;
export function createDuplexObservability(options: DuplexObservabilityOptions = {}): DuplexObservability {
const recorder = options.recorder ?? NOOP_DUPLEX_OBSERVABILITY_RECORDER;
const provider = normalizeDuplexProvider(options.providerKey);
const transport: DuplexTransportValue = options.transport ?? "duplex";
const safeSpan = (record: DuplexTelemetrySpanRecord): void => {
const safeSpan = (record: DuplexObservabilitySpanRecord): void => {
try {
recorder.recordSpan(record);
} catch {
// Observability must not break the request path.
}
};
const safeCounter = (record: DuplexTelemetryCounterRecord): void => {
const safeCounter = (record: DuplexObservabilityCounterRecord): void => {
try {
recorder.incrementCounter(record);
} catch {
// Observability must not break the request path.
}
};
const safeEvent = (record: DuplexTelemetryEventRecord): void => {
const safeEvent = (record: DuplexObservabilityEventRecord): void => {
try {
recorder.emitEvent(record);
} catch {
@@ -313,7 +382,7 @@ export function createDuplexTelemetry(options: DuplexTelemetryOptions = {}): Dup
};
const recordFallback = (reason: DuplexFallbackReason): void => {
const dimensions: DuplexTelemetryDimensions = {
const dimensions: DuplexObservabilityDimensions = {
provider,
transport: "file",
outcome: "error",
@@ -330,9 +399,9 @@ export function createDuplexTelemetry(options: DuplexTelemetryOptions = {}): Dup
ready(): void {
if (settled) return;
settled = true;
const dimensions: DuplexTelemetryDimensions = {
const dimensions: DuplexObservabilityDimensions = {
provider,
transport: "duplex",
transport,
outcome: "ok",
};
safeSpan({ name: DUPLEX_SPAN_CHANNEL_OPEN, dimensions });
@@ -349,7 +418,7 @@ export function createDuplexTelemetry(options: DuplexTelemetryOptions = {}): Dup
// closed dimension key, so no new key reaches a sink.
safeSpan({
name: DUPLEX_SPAN_CHANNEL_OPEN,
dimensions: { provider, transport: "duplex", outcome: "error", fallback_reason: reason },
dimensions: { provider, transport, outcome: "error", fallback_reason: reason },
});
recordFallback(reason);
},
@@ -359,14 +428,14 @@ export function createDuplexTelemetry(options: DuplexTelemetryOptions = {}): Dup
recordRequest(record: { latencyMs: number; outcome: DuplexOutcomeValue }): void {
safeSpan({
name: DUPLEX_SPAN_REQUEST,
dimensions: { provider, transport: "duplex", outcome: record.outcome },
dimensions: { provider, transport, outcome: record.outcome },
latencyMs: record.latencyMs,
});
},
recordLoss(lossClass: DuplexLossClass, lossReason: DuplexLossReason): void {
const dimensions: DuplexTelemetryDimensions = {
const dimensions: DuplexObservabilityDimensions = {
provider,
transport: "duplex",
transport,
outcome: "error",
loss_class: lossClass,
loss_reason: lossReason,
@@ -377,7 +446,7 @@ export function createDuplexTelemetry(options: DuplexTelemetryOptions = {}): Dup
recordSessionLeak(): void {
safeCounter({
metric: DUPLEX_COUNTER_SESSION_LEAK_TOTAL,
dimensions: { provider, transport: "duplex", outcome: "error" },
dimensions: { provider, transport, outcome: "error" },
});
},
};
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,901 @@
/**
* Host HTTP/2 server for the sandbox callback bridge transport.
*
* The server wraps one {@link CommandManagedDuplexChannel} as a Node `Duplex`
* and runs one plaintext HTTP/2 session on it. It maps every stream on that
* session to one call of the caller-supplied `forwardRequest` handler, then
* writes the result back as the stream response. The handler applies the real
* host token and the run attribution, so those rules stay in one place, next
* to the existing file-bridge and duplex-bridge forward path.
*
* This file does not select the transport for a run. It builds and tests the
* host half of the pair in isolation; a later phase wires the pair into the
* transport-selection path.
*
* Requests flow from the sandbox to the host only: the host never opens a
* stream to the sandbox. The server enforces three checks, in this order, for
* every stream:
* 1. a constant-time compare of the bridge token against the per-run token
* (accepted security fix 4), before any other processing;
* 2. one canonical parse of the `:path` pseudo-header (accepted security fix
* 3), whose result feeds both the route allowlist and the forward URL;
* 3. the route allowlist and the header allowlist, reused unchanged from
* `sandbox-callback-bridge.ts`.
* The server also bounds ten `http2.createServer` options (accepted security
* fix 1), so Node enforces the session, header, and stream-reset limits on
* every connection with no new component.
*/
import { Duplex } from "node:stream";
import http2 from "node:http2";
import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
import {
authorizeSandboxCallbackBridgeRequestWithRoutes,
compareBridgeTokensConstantTime,
sanitizeSandboxCallbackBridgeHeaders,
DEFAULT_SANDBOX_CALLBACK_BRIDGE_HEADER_ALLOWLIST,
DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES,
DEFAULT_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST,
type SandboxCallbackBridgeRouteRule,
} from "./sandbox-callback-bridge.js";
// ---------------------------------------------------------------------------
// Bounded server options (accepted security fix 1). Node enforces each value,
// so naming them adds configuration and no new component. Every value and
// every name below matches the board-approved table exactly.
// ---------------------------------------------------------------------------
/** Server push. The transport never needs it. */
export const HTTP2_BRIDGE_ENABLE_PUSH = false;
/** Open streams. This matches the current broker limit ({@link DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS} in `duplex-bridge-broker.ts`). */
export const HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS = 64;
/** One decompressed header list. The Node default is 65535. */
export const HTTP2_BRIDGE_MAX_HEADER_LIST_SIZE = 16384;
/** The header-compression table. This keeps the Node default. */
export const HTTP2_BRIDGE_HEADER_TABLE_SIZE = 4096;
/** Session memory in mebibytes. The Node default is 10. */
export const HTTP2_BRIDGE_MAX_SESSION_MEMORY = 16;
/** Header pairs per request. This names the Node default. */
export const HTTP2_BRIDGE_MAX_HEADER_LIST_PAIRS = 128;
/** The outbound compression table. */
export const HTTP2_BRIDGE_MAX_DEFLATE_DYNAMIC_TABLE_SIZE = 4096;
/** Invalid frames before Node closes the session. */
export const HTTP2_BRIDGE_MAX_SESSION_INVALID_FRAMES = 100;
/** Rejected streams before Node closes the session. */
export const HTTP2_BRIDGE_MAX_SESSION_REJECTED_STREAMS = 100;
/** The stream-reset budget (frames per interval). Node sends GOAWAY past the budget. */
export const HTTP2_BRIDGE_STREAM_RESET_RATE = 10;
/** The stream-reset budget (burst allowance). Node sends GOAWAY past the budget. */
export const HTTP2_BRIDGE_STREAM_RESET_BURST = 100;
/**
* The full bounded options object. The server passes this object, unchanged,
* to `http2.createServer`. A test asserts every value on this object, so it
* proves the running server actually carries the bound, not only that the
* named constant exists.
*/
export const HTTP2_BRIDGE_SERVER_OPTIONS: http2.ServerOptions = {
settings: {
enablePush: HTTP2_BRIDGE_ENABLE_PUSH,
maxConcurrentStreams: HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS,
maxHeaderListSize: HTTP2_BRIDGE_MAX_HEADER_LIST_SIZE,
headerTableSize: HTTP2_BRIDGE_HEADER_TABLE_SIZE,
},
maxSessionMemory: HTTP2_BRIDGE_MAX_SESSION_MEMORY,
maxHeaderListPairs: HTTP2_BRIDGE_MAX_HEADER_LIST_PAIRS,
maxDeflateDynamicTableSize: HTTP2_BRIDGE_MAX_DEFLATE_DYNAMIC_TABLE_SIZE,
maxSessionInvalidFrames: HTTP2_BRIDGE_MAX_SESSION_INVALID_FRAMES,
maxSessionRejectedStreams: HTTP2_BRIDGE_MAX_SESSION_REJECTED_STREAMS,
streamResetRate: HTTP2_BRIDGE_STREAM_RESET_RATE,
streamResetBurst: HTTP2_BRIDGE_STREAM_RESET_BURST,
};
// ---------------------------------------------------------------------------
// Duplex channel adapter
// ---------------------------------------------------------------------------
/** The default cap, in bytes, on the read-side queue {@link wrapDuplexChannelAsNodeDuplex}
* holds once `Duplex.push()` reports the readable side is full (a `false`
* return). A sandbox-controlled channel has no upstream pause: `onData` below
* keeps delivering bytes whether or not the HTTP/2 session keeps up with
* them. Past this cap the wrapper treats the channel as stuck, not merely
* slow, and fails closed: it stops the channel and destroys the `Duplex`, so
* a producer that keeps outpacing its reader cannot grow host memory without
* bound. This cap also bounds one single chunk: the wrapper checks a chunk's
* own size against it before `push()` ever runs, so one oversized chunk
* cannot cross the cap on its first delivery, before the queue holds
* anything to compare it against. */
export const DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES = HTTP2_BRIDGE_MAX_SESSION_MEMORY * 1024 * 1024;
/** The default bound, in milliseconds, on how long the read-side queue
* {@link wrapDuplexChannelAsNodeDuplex} holds can stay non-empty with no
* chunk draining from it. The byte cap above bounds how much memory a stuck
* reader can hold; it does not bound how long the reader can stay stuck. A
* consumer that never resumes reading would otherwise hold the channel open,
* backpressured, for as long as the queue stays under the byte cap. Each
* drained chunk renews this bound, so a consumer that keeps making real
* progress never trips it; only a consumer that stops resuming entirely
* does. */
export const DEFAULT_HTTP2_BRIDGE_READ_BACKPRESSURE_STALL_MS = 30_000;
/**
* Wrap a {@link CommandManagedDuplexChannel} as a Node `Duplex`, so an
* `Http2Server` can run one session directly on it (`server.emit("connection",
* duplex)`). The wrapper never buffers more than one write in flight: it calls
* the stream write callback only after the channel's own write call settles
* (the backpressure constraint), never as a delivery signal. The provider
* accepts many megabytes in milliseconds and holds them in its own buffer, so
* this direction stays governed by the channel's own write-settle timing.
*
* The read direction needs its own bound. The channel exposes no pause: once
* `onData` below is registered, the channel keeps calling it for every byte
* the sandbox sends, with no way for this wrapper to slow it down. Node's
* `Duplex.push()` reports back-pressure through its boolean return, not by
* refusing the call, so a caller that ignores a `false` return and keeps
* pushing grows the readable side's internal buffer with no limit. This
* wrapper honors that signal instead: while `push()` reports room, it pushes
* directly; once `push()` reports the readable side is full, it queues each
* later chunk instead of pushing past that signal, and drains the queue from
* `read()`, which Node calls again only once the consumer wants more. Every
* chunk, on either path, first checks against
* {@link DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES} (or the caller's
* `maxBufferedReadBytes`) on its own size, and the queue checks against the
* same cap on its cumulative size: past either check the wrapper fails
* closed instead of buffering further, because the channel has no pause to
* fall back on. A second, independent bound —
* {@link DEFAULT_HTTP2_BRIDGE_READ_BACKPRESSURE_STALL_MS} (or the caller's
* `readBackpressureStallMs`) — covers the case the byte cap does not: a
* consumer that stops reading entirely, so the queue never grows past the
* byte cap but also never drains. This bound renews on every chunk the
* queue drains, so a consumer that keeps making real progress never trips
* it.
*/
export function wrapDuplexChannelAsNodeDuplex(
channel: CommandManagedDuplexChannel,
options: { maxBufferedReadBytes?: number; readBackpressureStallMs?: number } = {},
): Duplex {
const maxBufferedReadBytes = options.maxBufferedReadBytes ?? DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES;
const readBackpressureStallMs =
options.readBackpressureStallMs ?? DEFAULT_HTTP2_BRIDGE_READ_BACKPRESSURE_STALL_MS;
// Chunks `onData` already delivered that `push()` has not yet accepted,
// in arrival order. `read()` drains this queue before it lets Node pull
// any new bytes, so the delivery order the channel used stays intact.
const pendingReads: Buffer[] = [];
let pendingReadBytes = 0;
// True once `push()` last reported room for more, or before the first
// push call. `onData` pushes directly while this holds; once a `push()`
// call reports no room, later chunks queue in `pendingReads` instead.
let canPushMore = true;
// True once the channel exited. `endReadableIfDrained` pushes `null` only
// after the queue this wrapper still holds fully drains, so a chunk that
// arrived before the exit is never dropped.
let channelExited = false;
// Arms while the queue holds at least one chunk; clears once it fully
// drains. Fires `readBackpressureStallMs` after the queue's last drain (or
// its first chunk, if it never drained at all) with no further drain, so a
// consumer that stops resuming does not hold the channel open forever
// under the byte cap.
let backpressureStallTimer: ReturnType<typeof setTimeout> | undefined;
function clearBackpressureStallTimer(): void {
if (backpressureStallTimer === undefined) return;
clearTimeout(backpressureStallTimer);
backpressureStallTimer = undefined;
}
function armBackpressureStallTimer(): void {
clearBackpressureStallTimer();
backpressureStallTimer = setTimeout(() => {
failClosed(
"Sandbox HTTP/2 channel's read backpressure queue did not drain within the stall bound; the reader appears stuck.",
);
}, readBackpressureStallMs);
backpressureStallTimer.unref?.();
}
function failClosed(message: string): void {
clearBackpressureStallTimer();
channel.stop();
duplex.destroy(new Error(message));
}
function endReadableIfDrained(): void {
if (!channelExited || pendingReads.length > 0 || duplex.destroyed) return;
duplex.push(null);
}
const duplex: Duplex = new Duplex({
read() {
canPushMore = true;
let drainedAChunk = false;
while (canPushMore && pendingReads.length > 0) {
const next = pendingReads.shift();
if (next === undefined) break;
pendingReadBytes -= next.byteLength;
drainedAChunk = true;
canPushMore = duplex.push(next);
}
if (pendingReads.length === 0) {
clearBackpressureStallTimer();
} else if (drainedAChunk) {
// The queue still holds chunks, but at least one drained just now:
// real progress, so the stall bound renews instead of expiring under
// a consumer that is still reading, only slowly.
armBackpressureStallTimer();
}
endReadableIfDrained();
},
write(chunk: unknown, _encoding, callback) {
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as ArrayBufferLike);
let settleResult: unknown;
try {
settleResult = channel.write(bytes);
} catch (error) {
callback(error instanceof Error ? error : new Error(String(error)));
return;
}
if (
settleResult != null &&
typeof (settleResult as Promise<void>).then === "function"
) {
(settleResult as Promise<void>).then(
() => callback(),
(error) => callback(error instanceof Error ? error : new Error(String(error))),
);
} else {
callback();
}
},
final(callback) {
channel
.close()
.then(() => callback(), (error) => callback(error instanceof Error ? error : new Error(String(error))));
},
});
channel.onData((chunk) => {
if (duplex.destroyed) return;
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
// Check one chunk's own size against the cap before either path below
// runs. `push()` never refuses a call on its size, so a single chunk
// larger than the whole cap would otherwise reach Node's internal
// buffer unbounded on the direct-push path, before the queue this
// wrapper owns ever holds anything to compare a later chunk against.
if (bytes.byteLength > maxBufferedReadBytes) {
failClosed(
"Sandbox HTTP/2 channel delivered one chunk larger than the bounded read backpressure buffer.",
);
return;
}
if (canPushMore && pendingReads.length === 0) {
canPushMore = duplex.push(bytes);
return;
}
// The readable side already reported it is full, and the channel has no
// pause to slow the sandbox side down: queue this chunk instead of
// pushing past that signal. Bound the queue, so a producer that keeps
// outpacing its reader cannot grow it without limit.
if (pendingReads.length === 0) {
// The queue was empty until this chunk: arm the stall bound, so a
// consumer that never resumes reading still ends the channel, even
// though this chunk alone stays under the byte cap.
armBackpressureStallTimer();
}
pendingReadBytes += bytes.byteLength;
if (pendingReadBytes > maxBufferedReadBytes) {
failClosed(
"Sandbox HTTP/2 channel exceeded the bounded read backpressure buffer; the reader could not keep up.",
);
return;
}
pendingReads.push(bytes);
});
channel.onExit(() => {
channelExited = true;
endReadableIfDrained();
});
// The channel exited, or `read()`/`onData` above failed the `Duplex`
// closed: either way, no further chunk will ever drain, so the stall
// timer serves no purpose and only holds a stray handle open.
duplex.once("close", clearBackpressureStallTimer);
return duplex;
}
// ---------------------------------------------------------------------------
// Canonical `:path` parsing (accepted security fix 3)
// ---------------------------------------------------------------------------
/** The reason {@link parseCanonicalBridgeRequestPath} rejected one request. */
export type CanonicalBridgeRequestPathRejection =
| "missing_path"
| "duplicate_pseudo_header"
| "non_origin_form"
| "encoded_slash"
| "backslash"
| "nul_byte"
| "dot_segment";
/** The one parsed pathname and query. Both the route allowlist and the forward
* URL builder read this same value; the host never parses `:path` twice. */
export interface CanonicalBridgeRequestPath {
pathname: string;
/** The query string, in `URL.search` form: empty, or a leading `?`. */
query: string;
}
export type CanonicalBridgeRequestPathResult =
| { ok: true; value: CanonicalBridgeRequestPath }
| { ok: false; reason: CanonicalBridgeRequestPathRejection };
/** The request pseudo-headers HTTP/2 allows exactly one of, per request. */
const REQUEST_PSEUDO_HEADER_NAMES = [":method", ":scheme", ":authority", ":path"] as const;
/**
* Parse the `:path` pseudo-header exactly one time. The caller passes the
* returned pathname and query to both the route allowlist and the forward URL
* builder — never a second, independent parse of the raw header.
*
* The parser rejects a request that carries any of: a duplicate pseudo-header,
* a missing or empty `:path`, a non-origin-form path, an encoded slash, a
* backslash, a NUL byte, or a dot segment (checked before URL normalization
* would silently remove it, and after percent-decoding each segment, so an
* encoded dot segment cannot slip through).
*/
export function parseCanonicalBridgeRequestPath(
headers: http2.IncomingHttpHeaders,
): CanonicalBridgeRequestPathResult {
for (const name of REQUEST_PSEUDO_HEADER_NAMES) {
if (Array.isArray((headers as Record<string, unknown>)[name])) {
return { ok: false, reason: "duplicate_pseudo_header" };
}
}
const rawPath = headers[":path"];
if (typeof rawPath !== "string" || rawPath.length === 0) {
return { ok: false, reason: "missing_path" };
}
// Origin-form only: a single leading "/", never "//" (network-path form) and
// never an absolute-form URI ("scheme://...").
if (!rawPath.startsWith("/") || rawPath.startsWith("//") || rawPath.includes("://")) {
return { ok: false, reason: "non_origin_form" };
}
if (/%2f/i.test(rawPath)) {
return { ok: false, reason: "encoded_slash" };
}
if (rawPath.includes("\\")) {
return { ok: false, reason: "backslash" };
}
if (rawPath.includes("\0") || /%00/i.test(rawPath)) {
return { ok: false, reason: "nul_byte" };
}
const queryIndex = rawPath.indexOf("?");
const rawPathname = queryIndex === -1 ? rawPath : rawPath.slice(0, queryIndex);
for (const segment of rawPathname.split("/")) {
let decoded: string;
try {
decoded = decodeURIComponent(segment);
} catch {
return { ok: false, reason: "non_origin_form" };
}
if (decoded === "." || decoded === "..") {
return { ok: false, reason: "dot_segment" };
}
}
// Every raw-string check above passed, and the path carries no dot segment,
// so `URL` normalization here changes nothing but percent-encoding; it stays
// safe to build the canonical pathname and query from it.
let url: URL;
try {
url = new URL(rawPath, "http://bridge.internal");
} catch {
return { ok: false, reason: "non_origin_form" };
}
return { ok: true, value: { pathname: url.pathname, query: url.search } };
}
/**
* Build the forward URL from the one canonical parse. This mirrors
* `buildBridgeForwardUrl` in `execution-target.ts`, which the file bridge and
* the duplex bridge use today; a later phase wires the HTTP/2 host handler to
* that same forward path and can consolidate the two into one export.
*/
export function buildHttp2BridgeForwardUrl(
baseUrl: string,
request: CanonicalBridgeRequestPath,
): URL {
const url = new URL(request.pathname, baseUrl);
const query = request.query.trim();
url.search = query.startsWith("?") ? query.slice(1) : query;
return url;
}
// ---------------------------------------------------------------------------
// PING stall detection
// ---------------------------------------------------------------------------
/** The default interval between two liveness PING frames, in milliseconds. */
export const DEFAULT_HTTP2_BRIDGE_PING_INTERVAL_MS = 5_000;
/** The default bound a sent PING waits for its ack before the session counts as stalled. */
export const DEFAULT_HTTP2_BRIDGE_PING_STALL_MS = 20_000;
/** The default idle bound on a request body read: the maximum gap between
* two received chunks (or between the token check and the first chunk)
* before the server treats the stream as stalled. Each received chunk resets
* this bound, so a slow peer that keeps making real progress completes; only
* a peer that stops sending trips it. */
export const DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_TIMEOUT_MS = 30_000;
/** The default hard ceiling on a request body read's total lifetime: an
* absolute bound armed once, at the start of the read, and never renewed by
* later progress. This bound is independent of
* {@link DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_TIMEOUT_MS}: the idle bound
* resets on every chunk to catch a peer that stops sending; this ceiling
* catches a peer that never stops sending but also never finishes, so a
* peer cannot use a steady trickle of small chunks to hold a
* {@link HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS} stream slot open forever. Set
* well above {@link DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_TIMEOUT_MS} so a
* legitimate upload that makes real but slow progress — a chunk every few
* seconds, well inside the idle bound — still has room to finish. */
export const DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_LIFETIME_CEILING_MS = 480_000;
/** The default bound {@link Http2BridgeServerHandle.close} waits for an
* active session to close on its own before it force-destroys the session. A
* session that carries a stalled stream would otherwise hold `close()` open
* forever, because `session.close()` waits for every open stream to end. */
export const DEFAULT_HTTP2_BRIDGE_CLOSE_GRACE_MS = 5_000;
function startHttp2BridgePingWatchdog(
session: http2.ServerHttp2Session,
input: { intervalMs: number; stallMs: number; onStall: (error: Error) => void },
): () => void {
let stopped = false;
let pingTimer: ReturnType<typeof setTimeout> | undefined;
let stallTimer: ReturnType<typeof setTimeout> | undefined;
function sendOnePing(): void {
if (stopped) return;
stallTimer = setTimeout(() => {
if (stopped) return;
stopped = true;
input.onStall(new Error("HTTP/2 bridge session stalled: no PING ack within the stall bound."));
}, input.stallMs);
stallTimer.unref?.();
try {
session.ping((error) => {
if (stopped) return;
if (stallTimer) clearTimeout(stallTimer);
if (error) {
stopped = true;
input.onStall(error instanceof Error ? error : new Error(String(error)));
return;
}
pingTimer = setTimeout(sendOnePing, input.intervalMs);
pingTimer.unref?.();
});
} catch (error) {
if (stallTimer) clearTimeout(stallTimer);
stopped = true;
input.onStall(error instanceof Error ? error : new Error(String(error)));
}
}
pingTimer = setTimeout(sendOnePing, input.intervalMs);
pingTimer.unref?.();
return () => {
stopped = true;
if (pingTimer) clearTimeout(pingTimer);
if (stallTimer) clearTimeout(stallTimer);
};
}
// ---------------------------------------------------------------------------
// The server
// ---------------------------------------------------------------------------
/** The result of one forward call. The server turns it into one stream response. */
export interface Http2BridgeForwardResult {
status: number;
headers?: Record<string, string>;
body?: Buffer | string;
}
/**
* The one canonically-parsed, route-authorized, header-sanitized request the
* server hands to the forward handler.
*/
export interface Http2BridgeForwardRequest {
method: string;
pathname: string;
query: string;
headers: Record<string, string>;
body: Buffer;
}
export type Http2BridgeForwardHandler = (
request: Http2BridgeForwardRequest,
) => Promise<Http2BridgeForwardResult>;
/** The GOAWAY the server observed, naming the last stream ID the peer processed. */
export interface Http2BridgeGoawayRecord {
lastStreamId: number;
errorCode: number;
}
/** Classify one stream ID against an observed GOAWAY's last processed stream ID. */
export function classifyStreamAgainstGoaway(
streamId: number,
lastStreamId: number,
): "accepted" | "not_accepted" {
return streamId <= lastStreamId ? "accepted" : "not_accepted";
}
export interface CreateHttp2BridgeServerOptions {
/** The per-run bridge token. The server compares it, constant-time, against
* the token on every stream before route or header processing. */
bridgeToken: string;
/** The forward handler the server calls for each authorized request. */
forwardRequest: Http2BridgeForwardHandler;
/** The route allowlist. The default is {@link DEFAULT_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST}. */
routes?: readonly SandboxCallbackBridgeRouteRule[];
/** The header allowlist. The default is {@link DEFAULT_SANDBOX_CALLBACK_BRIDGE_HEADER_ALLOWLIST}. */
headerAllowlist?: readonly string[];
/** The maximum request body size, in bytes. The default is {@link DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES}. */
maxBodyBytes?: number;
/** The interval between two liveness PING frames, in milliseconds. */
pingIntervalMs?: number;
/** The bound a sent PING waits for its ack before the server closes the session. */
pingStallMs?: number;
/** The idle bound on a request body read: the maximum gap between two
* received chunks. The default is
* {@link DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_TIMEOUT_MS}. */
requestBodyTimeoutMs?: number;
/** The hard ceiling on a request body read's total lifetime, armed once
* and never renewed by progress. See
* {@link DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_LIFETIME_CEILING_MS} for the
* default and the reasoning behind it. */
requestBodyLifetimeCeilingMs?: number;
/** The bound {@link Http2BridgeServerHandle.close} waits for an active
* session to close on its own before it force-destroys the session. The
* default is {@link DEFAULT_HTTP2_BRIDGE_CLOSE_GRACE_MS}. */
closeGraceMs?: number;
/** The cap, in bytes, on data this server holds once a bound `Duplex`
* reports its readable side is full (`push()` returns `false`). Past this
* cap the server treats the channel as stuck, not merely slow: see
* {@link wrapDuplexChannelAsNodeDuplex}. The default is
* {@link DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES}. */
maxBufferedReadBytes?: number;
/** The bound, in milliseconds, on how long the read backpressure queue
* {@link wrapDuplexChannelAsNodeDuplex} holds can stay non-empty with no
* chunk draining from it. The default is
* {@link DEFAULT_HTTP2_BRIDGE_READ_BACKPRESSURE_STALL_MS}. */
readBackpressureStallMs?: number;
/** The sink for a GOAWAY the server observed on a session (one the sandbox
* side sent to the host). */
onGoaway?: (record: Http2BridgeGoawayRecord) => void;
/** The sink for a session-level fault (a stall, a protocol fault). */
onSessionError?: (error: Error) => void;
/**
* Fires once for each new session. A caller uses the live
* `ServerHttp2Session` to send its own GOAWAY (accepted security fix's
* GOAWAY-classification behavior is meaningful only from the side that
* names the last stream it processed — the host, since every stream
* originates from the sandbox). A test uses this hook to drive the
* GOAWAY test deterministically.
*/
onSession?: (session: http2.ServerHttp2Session) => void;
}
/** The handle {@link createHttp2BridgeServer} returns. */
export interface Http2BridgeServerHandle {
/** The underlying `Http2Server`. It is never `listen()`-ed; every session
* binds through {@link Http2BridgeServerHandle.bindChannel}. */
readonly server: http2.Http2Server;
/** Wrap the channel as a `Duplex` and run one HTTP/2 session on it. Returns
* the wrapped `Duplex`, so a caller can also drive it directly (a test uses
* this to bind one side of a paired in-memory `Duplex`). */
bindChannel(channel: CommandManagedDuplexChannel): Duplex;
/** Close every active session. Safe to call more than one time. */
close(): Promise<void>;
}
function normalizeStreamMethod(value: string | string[] | undefined): string {
return typeof value === "string" && value.trim().length > 0 ? value.trim().toUpperCase() : "GET";
}
function readBridgeTokenHeader(headers: http2.IncomingHttpHeaders): string | undefined {
const raw = headers.authorization;
if (typeof raw !== "string" || !raw.startsWith("Bearer ")) return undefined;
return raw.slice("Bearer ".length);
}
function toOutboundHeaderRecord(headers: http2.IncomingHttpHeaders): Record<string, string> {
const out: Record<string, string> = {};
for (const [key, value] of Object.entries(headers)) {
if (key.startsWith(":") || value == null) continue;
out[key.toLowerCase()] = Array.isArray(value) ? value.join(", ") : String(value);
}
return out;
}
/** The size and time bounds a request body read enforces. */
export interface Http2BridgeBodyBounds {
/** The maximum request body size, in bytes. */
maxBodyBytes: number;
/** The idle bound: see {@link DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_TIMEOUT_MS}. */
idleTimeoutMs: number;
/** The lifetime ceiling: see {@link DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_LIFETIME_CEILING_MS}. */
lifetimeCeilingMs: number;
}
/**
* Read one request body, bounded on size and on two independent time
* bounds: an idle bound and a total-lifetime ceiling. See
* {@link DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_TIMEOUT_MS} and
* {@link DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_LIFETIME_CEILING_MS} for why the
* server enforces each one.
*
* The `close` listener is the settle-of-last-resort: it fires whenever the
* stream ends for any reason at all — a normal end, an error, a timeout- or
* shutdown-triggered `destroy()`, or a peer reset — so the promise always
* settles and the caller never awaits a stream that already went away.
*/
function readHttp2StreamBody(
stream: http2.ServerHttp2Stream,
bounds: Http2BridgeBodyBounds,
): Promise<Buffer> {
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
let totalBytes = 0;
let settled = false;
let idleTimer: ReturnType<typeof setTimeout>;
let lifetimeCeilingTimer: ReturnType<typeof setTimeout>;
const settle = (run: () => void) => {
if (settled) return;
settled = true;
clearTimeout(idleTimer);
clearTimeout(lifetimeCeilingTimer);
run();
};
const armIdleTimer = () => {
clearTimeout(idleTimer);
idleTimer = setTimeout(() => {
settle(() => reject(new Error("Bridge request body stalled before it completed.")));
stream.destroy();
}, bounds.idleTimeoutMs);
idleTimer.unref?.();
};
// Arms one time, when the read starts, and never rearms on a chunk: see
// DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_LIFETIME_CEILING_MS for why the
// ceiling must stay independent of progress.
lifetimeCeilingTimer = setTimeout(() => {
settle(() => reject(new Error("Bridge request body passed the total lifetime ceiling.")));
stream.destroy();
}, bounds.lifetimeCeilingMs);
lifetimeCeilingTimer.unref?.();
armIdleTimer();
stream.on("data", (chunk: Buffer) => {
totalBytes += chunk.byteLength;
if (totalBytes > bounds.maxBodyBytes) {
settle(() => reject(new Error("Bridge request body exceeded the configured size limit.")));
stream.destroy();
return;
}
chunks.push(chunk);
// The chunk is real progress, so the peer is not stalled: reset the
// idle bound. The lifetime ceiling timer above does not reset here.
armIdleTimer();
});
stream.once("end", () => settle(() => resolve(Buffer.concat(chunks))));
stream.once("error", (error) => settle(() => reject(error instanceof Error ? error : new Error(String(error)))));
stream.once("aborted", () => settle(() => reject(new Error("Bridge request stream aborted."))));
stream.once("close", () => settle(() => reject(new Error("Bridge request stream closed before it completed."))));
});
}
function respondJson(stream: http2.ServerHttp2Stream, status: number, body: unknown): void {
if (stream.destroyed || stream.closed) return;
try {
stream.respond({ ":status": status, "content-type": "application/json" });
stream.end(JSON.stringify(body));
} catch {
// The peer reset the stream (RST_STREAM) before the server could answer.
// That fault stays local to this one stream; every other stream and the
// session itself stay unaffected.
}
}
/**
* Answer a denied stream, then consume and discard its request body under
* the same bounds ({@link Http2BridgeBodyBounds}) an authenticated request
* gets. A denied request's body content never reaches the forward handler,
* but the inbound half of the stream still needs a bound: without one, a
* peer that leaves the body unfinished keeps the stream open, holding one
* of the {@link HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS} slots for as long as
* it chooses. Nothing awaits the discard; the caller has already answered
* the request and moves on to the next stream.
*/
function denyRequest(
stream: http2.ServerHttp2Stream,
status: number,
body: unknown,
bounds: Http2BridgeBodyBounds,
): void {
respondJson(stream, status, body);
if (stream.destroyed || stream.closed) return;
readHttp2StreamBody(stream, bounds).catch(() => {
// The idle or lifetime bound above already destroyed the stream, or the
// peer reset it first. Either way the slot is free; the discarded body
// content is irrelevant to a denial.
});
}
/**
* Create the host HTTP/2 bridge server. The server runs no listener of its
* own: a caller wraps one duplex channel through {@link Http2BridgeServerHandle.bindChannel}
* per sandbox session.
*/
export function createHttp2BridgeServer(options: CreateHttp2BridgeServerOptions): Http2BridgeServerHandle {
const routes = options.routes ?? DEFAULT_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST;
const headerAllowlist = options.headerAllowlist ?? DEFAULT_SANDBOX_CALLBACK_BRIDGE_HEADER_ALLOWLIST;
const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES;
const pingIntervalMs = options.pingIntervalMs ?? DEFAULT_HTTP2_BRIDGE_PING_INTERVAL_MS;
const pingStallMs = options.pingStallMs ?? DEFAULT_HTTP2_BRIDGE_PING_STALL_MS;
const requestBodyTimeoutMs = options.requestBodyTimeoutMs ?? DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_TIMEOUT_MS;
const requestBodyLifetimeCeilingMs =
options.requestBodyLifetimeCeilingMs ?? DEFAULT_HTTP2_BRIDGE_REQUEST_BODY_LIFETIME_CEILING_MS;
const closeGraceMs = options.closeGraceMs ?? DEFAULT_HTTP2_BRIDGE_CLOSE_GRACE_MS;
const maxBufferedReadBytes = options.maxBufferedReadBytes ?? DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES;
const readBackpressureStallMs =
options.readBackpressureStallMs ?? DEFAULT_HTTP2_BRIDGE_READ_BACKPRESSURE_STALL_MS;
// Built one time and passed to every readHttp2StreamBody() and
// denyRequest() call below, so every stream on this server enforces the
// same bounds.
const bodyBounds: Http2BridgeBodyBounds = {
maxBodyBytes,
idleTimeoutMs: requestBodyTimeoutMs,
lifetimeCeilingMs: requestBodyLifetimeCeilingMs,
};
const server = http2.createServer(HTTP2_BRIDGE_SERVER_OPTIONS);
const activeSessions = new Set<http2.ServerHttp2Session>();
async function handleStream(
stream: http2.ServerHttp2Stream,
headers: http2.IncomingHttpHeaders,
): Promise<void> {
// Accepted security fix 4: the constant-time bridge-token compare runs
// before route processing and before header processing. This host check
// is independent of the gateway's own token check on the sandbox side.
if (!compareBridgeTokensConstantTime(options.bridgeToken, readBridgeTokenHeader(headers))) {
denyRequest(stream, 401, { error: "Invalid bridge token." }, bodyBounds);
return;
}
// Accepted security fix 3: parse `:path` exactly one time; both the route
// allowlist and the forward request below read this one result.
const parsedPath = parseCanonicalBridgeRequestPath(headers);
if (!parsedPath.ok) {
denyRequest(stream, 400, { error: `Invalid request path: ${parsedPath.reason}` }, bodyBounds);
return;
}
const method = normalizeStreamMethod(headers[":method"]);
const denialReason = authorizeSandboxCallbackBridgeRequestWithRoutes(
{ method, path: parsedPath.value.pathname },
routes,
);
if (denialReason) {
denyRequest(stream, 403, { error: denialReason }, bodyBounds);
return;
}
const sanitizedHeaders = sanitizeSandboxCallbackBridgeHeaders(
toOutboundHeaderRecord(headers),
headerAllowlist,
);
let body: Buffer;
try {
body = await readHttp2StreamBody(stream, bodyBounds);
} catch (error) {
respondJson(stream, 413, { error: error instanceof Error ? error.message : String(error) });
return;
}
let result: Http2BridgeForwardResult;
try {
result = await options.forwardRequest({
method,
pathname: parsedPath.value.pathname,
query: parsedPath.value.query,
headers: sanitizedHeaders,
body,
});
} catch (error) {
respondJson(stream, 502, { error: error instanceof Error ? error.message : String(error) });
return;
}
if (stream.destroyed || stream.closed) return;
const responseHeaders: http2.OutgoingHttpHeaders = { ":status": result.status };
for (const [key, value] of Object.entries(result.headers ?? {})) {
if (key.toLowerCase() === "content-length") continue;
responseHeaders[key] = value;
}
try {
stream.respond(responseHeaders);
stream.end(result.body);
} catch {
// The peer reset the stream (RST_STREAM) between dispatch and response.
// One stream's write fault stays local to that stream.
}
}
server.on("session", (session) => {
activeSessions.add(session);
options.onSession?.(session);
const stopWatchdog = startHttp2BridgePingWatchdog(session, {
intervalMs: pingIntervalMs,
stallMs: pingStallMs,
// Destroying the session with an error routes back through this same
// session's own `error` listener below, which reports it exactly once.
onStall: (error) => {
if (!session.destroyed) session.destroy(error);
},
});
session.on("goaway", (errorCode: number, lastStreamId: number) => {
options.onGoaway?.({ lastStreamId, errorCode });
});
session.on("close", () => {
stopWatchdog();
activeSessions.delete(session);
});
session.on("error", (error) => {
stopWatchdog();
options.onSessionError?.(error instanceof Error ? error : new Error(String(error)));
});
});
server.on("stream", (stream, headers) => {
void handleStream(stream, headers).catch((error) => {
// A fault inside `handleStream` itself (not a forward-handler or
// stream-body rejection, both already caught above) is a defensive
// last resort. Destroy only this stream; the session stays open.
if (!stream.destroyed) {
stream.destroy(error instanceof Error ? error : new Error(String(error)));
}
});
});
return {
server,
bindChannel(channel: CommandManagedDuplexChannel): Duplex {
const duplex = wrapDuplexChannelAsNodeDuplex(channel, { maxBufferedReadBytes, readBackpressureStallMs });
server.emit("connection", duplex);
return duplex;
},
async close(): Promise<void> {
// `session.close()` sends GOAWAY and waits for every open stream to end
// on its own; a stalled stream (its body never completes, and its own
// timeout has not yet fired) would hold this wait open forever. The
// grace timer bounds it: past `closeGraceMs`, the server force-destroys
// the session, which ends its streams at once and settles their body
// reads through the `close` backstop in `readHttp2StreamBody`.
await Promise.all(
[...activeSessions].map(
(session) =>
new Promise<void>((resolve) => {
if (session.closed || session.destroyed) {
resolve();
return;
}
const forceDestroyTimer = setTimeout(() => {
if (!session.destroyed) session.destroy();
}, closeGraceMs);
forceDestroyTimer.unref?.();
session.close(() => {
clearTimeout(forceDestroyTimer);
resolve();
});
}),
),
);
},
};
}
-2
View File
@@ -86,7 +86,6 @@ export type {
export { inferOpenAiCompatibleBiller } from "./billing.js";
export {
ADAPTER_LOGIN_PANEL_MODES,
ADAPTER_LOGIN_SANDBOX_TRANSPORTS,
ADAPTER_LOGIN_TIMEOUT_POLICIES,
ADAPTER_LOGIN_COMPLETION_CLAIMS,
assertValidAdapterLoginCapability,
@@ -94,7 +93,6 @@ export {
} from "./login-capability.js";
export type {
AdapterLoginPanelMode,
AdapterLoginSandboxTransport,
AdapterLoginTimeoutPolicy,
AdapterLoginCompletionClaim,
AdapterLoginPrompt,
@@ -10,7 +10,6 @@ import {
function validCapability(): AdapterLoginCapability {
return {
panelMode: "displayed_code",
sandboxTransport: "streamed_exec",
timeoutPolicy: "caller_bounded",
getCommand: () => "vendor login",
parsePrompt: () => null,
@@ -25,7 +24,6 @@ describe("assertValidAdapterLoginCapability", () => {
it("accepts a well-formed capability with every optional member set", () => {
const capability: AdapterLoginCapability = {
panelMode: "submitted_browser_code",
sandboxTransport: "pseudo_terminal",
timeoutPolicy: "fixed",
getCommand: () => "vendor setup-token",
parsePrompt: () => null,
@@ -48,11 +46,6 @@ describe("assertValidAdapterLoginCapability", () => {
expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/panelMode/);
});
it("rejects an unknown sandboxTransport", () => {
const bad = { ...validCapability(), sandboxTransport: "http" };
expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/sandboxTransport/);
});
it("rejects an unknown timeoutPolicy", () => {
const bad = { ...validCapability(), timeoutPolicy: "unbounded" };
expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/timeoutPolicy/);
@@ -18,14 +18,6 @@
export const ADAPTER_LOGIN_PANEL_MODES = ["displayed_code", "submitted_browser_code"] as const;
export type AdapterLoginPanelMode = (typeof ADAPTER_LOGIN_PANEL_MODES)[number];
/**
* The sandbox transport that the login command needs. `streamed_exec` runs the
* command over the streamed exec channel. `pseudo_terminal` runs the command on
* a real pseudo-terminal, because a pipe emits no login prompt.
*/
export const ADAPTER_LOGIN_SANDBOX_TRANSPORTS = ["streamed_exec", "pseudo_terminal"] as const;
export type AdapterLoginSandboxTransport = (typeof ADAPTER_LOGIN_SANDBOX_TRANSPORTS)[number];
/**
* The host-side timeout policy. `caller_bounded` lets the caller set the
* timeout. `fixed` binds the timeout to a fixed adapter value.
@@ -68,8 +60,6 @@ export interface AdapterLoginCompletionContext {
export interface AdapterLoginCapability {
/** The login panel mode. */
panelMode: AdapterLoginPanelMode;
/** The sandbox transport that the login command needs. */
sandboxTransport: AdapterLoginSandboxTransport;
/** The host-side timeout policy. */
timeoutPolicy: AdapterLoginTimeoutPolicy;
/** Returns the fixed, non-secret login command. */
@@ -122,11 +112,6 @@ export function assertValidAdapterLoginCapability(
`${prefix}: "panelMode" must be one of ${ADAPTER_LOGIN_PANEL_MODES.join(", ")}.`,
);
}
if (!isOneOf(ADAPTER_LOGIN_SANDBOX_TRANSPORTS, cap.sandboxTransport)) {
throw new Error(
`${prefix}: "sandboxTransport" must be one of ${ADAPTER_LOGIN_SANDBOX_TRANSPORTS.join(", ")}.`,
);
}
if (!isOneOf(ADAPTER_LOGIN_TIMEOUT_POLICIES, cap.timeoutPolicy)) {
throw new Error(
`${prefix}: "timeoutPolicy" must be one of ${ADAPTER_LOGIN_TIMEOUT_POLICIES.join(", ")}.`,
@@ -1,8 +1,8 @@
import { describe, expect, it } from "vitest";
import {
createSetupTokenPtyTransport,
type SetupTokenPtySession,
} from "./setup-token-transport.js";
createLoginPtyTransport,
type LoginPtySession,
} from "./login-pty-transport.js";
// The Enter byte the terminal login UI reads to submit the browser code. The
// login runner appends this byte to the code. The transport forwards the bytes
@@ -14,7 +14,7 @@ const ENTER = "\r";
* stream on demand, and records the stop and the close. The tests use it in
* place of a real pseudo-terminal, so the transport runs with no sandbox.
*/
function createFakePtySession(): SetupTokenPtySession & {
function createFakePtySession(): LoginPtySession & {
writes: string[];
emit: (chunk: string) => void;
finish: (exitCode: number | null) => void;
@@ -74,10 +74,10 @@ function createFakePtySession(): SetupTokenPtySession & {
};
}
describe("createSetupTokenPtyTransport", () => {
describe("createLoginPtyTransport", () => {
it("delivers delayed input to the process", async () => {
const session = createFakePtySession();
const transport = createSetupTokenPtyTransport(async () => session);
const transport = createLoginPtyTransport(async () => session);
const started = transport.start("claude setup-token", () => {});
await session.ready;
@@ -94,7 +94,7 @@ describe("createSetupTokenPtyTransport", () => {
it("returns incremental terminal output to the runner", async () => {
const session = createFakePtySession();
const received: string[] = [];
const transport = createSetupTokenPtyTransport(async () => session);
const transport = createLoginPtyTransport(async () => session);
const started = transport.start("claude setup-token", (chunk) => {
received.push(chunk);
@@ -114,7 +114,7 @@ describe("createSetupTokenPtyTransport", () => {
it("delivers the Enter byte after the browser code", async () => {
const session = createFakePtySession();
const transport = createSetupTokenPtyTransport(async () => session);
const transport = createLoginPtyTransport(async () => session);
const started = transport.start("claude setup-token", () => {});
await session.ready;
@@ -133,7 +133,7 @@ describe("createSetupTokenPtyTransport", () => {
it("resolves start with the child exit code", async () => {
const session = createFakePtySession();
const transport = createSetupTokenPtyTransport(async () => session);
const transport = createLoginPtyTransport(async () => session);
const started = transport.start("claude setup-token", () => {});
session.finish(7);
@@ -143,7 +143,7 @@ describe("createSetupTokenPtyTransport", () => {
it("stops the child with a direct kill", async () => {
const session = createFakePtySession();
const transport = createSetupTokenPtyTransport(async () => session);
const transport = createLoginPtyTransport(async () => session);
const started = transport.start("claude setup-token", () => {});
await session.ready;
@@ -156,7 +156,7 @@ describe("createSetupTokenPtyTransport", () => {
it("disposes the session resources", async () => {
const session = createFakePtySession();
const transport = createSetupTokenPtyTransport(async () => session);
const transport = createLoginPtyTransport(async () => session);
const started = transport.start("claude setup-token", () => {});
session.finish(0);
@@ -168,7 +168,7 @@ describe("createSetupTokenPtyTransport", () => {
it("stays safe when stop and dispose run before start", async () => {
const session = createFakePtySession();
const transport = createSetupTokenPtyTransport(async () => session);
const transport = createLoginPtyTransport(async () => session);
// The runner may stop or dispose before it starts the child. The transport
// must not throw, and it must not open a session for a run it already stopped.
@@ -185,7 +185,7 @@ describe("createSetupTokenPtyTransport", () => {
const openGate = new Promise<void>((resolve) => {
resolveOpen = resolve;
});
const transport = createSetupTokenPtyTransport(async () => {
const transport = createLoginPtyTransport(async () => {
await openGate;
return session;
});
@@ -207,7 +207,7 @@ describe("createSetupTokenPtyTransport", () => {
const openGate = new Promise<void>((resolve) => {
resolveOpen = resolve;
});
const transport = createSetupTokenPtyTransport(async () => {
const transport = createLoginPtyTransport(async () => {
await openGate;
return session;
});
@@ -1,4 +1,4 @@
// The setup-token pseudo-terminal transport. It gives the Claude `setup-token`
// The login pseudo-terminal (PTY) transport. It gives the Claude `setup-token`
// login runner a child that runs on a real pseudo-terminal (PTY). The command
// needs a PTY: pipe stdio emits no login prompt. The transport starts the
// command on a PTY, streams the incremental terminal output, delivers delayed
@@ -7,7 +7,7 @@
// This module is provider-agnostic and pure. It holds no Node built-in import,
// so the browser-safe root entry can re-export it. A sandbox provider (for
// example the Daytona plugin) opens the concrete pseudo-terminal through a
// {@link SetupTokenPtySessionOpener}. A unit test opens a fake pseudo-terminal
// {@link LoginPtySessionOpener}. A unit test opens a fake pseudo-terminal
// through the same opener.
//
// Boundary (ANSI and OSC 8): the transport forwards the raw terminal bytes
@@ -20,7 +20,7 @@
* raw terminal output, accepts delayed input, and stops the child. The session
* forwards the raw terminal bytes; it runs no ANSI or OSC 8 handling.
*/
export interface SetupTokenPtySession {
export interface LoginPtySession {
/**
* Registers the one output listener. The session streams each raw terminal
* output chunk to `listener`, in order, as the pseudo-terminal emits it.
@@ -49,7 +49,7 @@ export interface SetupTokenPtySession {
* Opens a pseudo-terminal session for `command`. A provider binds this to its
* sandbox. The transport calls it one time, on start.
*/
export type SetupTokenPtySessionOpener = (command: string) => Promise<SetupTokenPtySession>;
export type LoginPtySessionOpener = (command: string) => Promise<LoginPtySession>;
/**
* The child side of the setup-token run, in the shape the login runner needs.
@@ -60,7 +60,7 @@ export type SetupTokenPtySessionOpener = (command: string) => Promise<SetupToken
* accepts the transport with no adapter. The transport never imports the runner,
* so the runner package keeps its one-way dependency on this package.
*/
export interface SetupTokenPtyTransport {
export interface LoginPtyTransport {
/**
* Opens the pseudo-terminal session for `command` and streams the terminal
* output to `onData` in order. Resolves with the child exit code when the
@@ -86,15 +86,15 @@ export interface SetupTokenPtyTransport {
}
/**
* Creates a {@link SetupTokenPtyTransport} over `open`. The transport adapts a
* Creates a {@link LoginPtyTransport} over `open`. The transport adapts a
* pseudo-terminal session into the runner driver shape. It buffers an early
* input write, it honors an early stop, and it forwards the raw terminal bytes
* with no ANSI or OSC 8 handling.
*/
export function createSetupTokenPtyTransport(
open: SetupTokenPtySessionOpener,
): SetupTokenPtyTransport {
let session: SetupTokenPtySession | null = null;
export function createLoginPtyTransport(
open: LoginPtySessionOpener,
): LoginPtyTransport {
let session: LoginPtySession | null = null;
let started = false;
let stopped = false;
// Input that the runner writes before the session opens. The transport flushes
@@ -113,7 +113,7 @@ export function createSetupTokenPtyTransport(
return {
async start(command, onData): Promise<{ exitCode: number | null }> {
if (started) {
throw new Error("setup-token PTY transport already started.");
throw new Error("login PTY transport already started.");
}
started = true;
const opened = await open(command);
@@ -0,0 +1,38 @@
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
// This test pins the published subpath surface of `package.json`.
//
// `@paperclipai/server` is itself a published npm package. Its published
// build imports `@paperclipai/adapter-utils/duplex-observability` as a real
// npm dependency, not as a workspace link. Node resolves that import against
// `publishConfig.exports`, so the subpath must stay published there. An
// earlier revision of this file denied the subpath with an explicit `null`
// entry; that denial broke module resolution for the published server. The
// subpath now falls through to the wildcard entry, the same as every other
// file in the package.
interface PackageManifest {
exports: Record<string, unknown>;
publishConfig: {
exports: Record<string, unknown>;
};
}
const manifestPath = fileURLToPath(new URL("../package.json", import.meta.url));
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as PackageManifest;
describe("publishConfig publishes the duplex observability subpath", () => {
it("does not deny the subpath in publishConfig.exports", () => {
expect(manifest.publishConfig.exports).not.toHaveProperty("./duplex-observability");
});
it("keeps the wildcard entry in publishConfig.exports so the subpath resolves through it", () => {
expect(manifest.publishConfig.exports["./*"]).toBeDefined();
});
it("keeps the top-level wildcard export unchanged", () => {
expect(manifest.exports["./*"]).toBe("./src/*.ts");
});
});
@@ -1,5 +1,6 @@
import { execFile as execFileCallback } from "node:child_process";
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { execFile as execFileCallback, spawn } from "node:child_process";
import { createServer } from "node:http";
import { mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
@@ -475,6 +476,68 @@ describe("sandbox callback bridge", () => {
}
});
it("recovers from transient queue polling failures and keeps relaying requests", async () => {
// A single reset or slow sandbox exec used to unwind the poll loop into its
// terminal catch, killing the relay for the rest of the run. The loop must
// instead back off, retry, and still deliver requests queued after the
// failure window.
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-transient-"));
cleanupDirs.push(rootDir);
const queueDir = path.posix.join(rootDir, "queue");
const directories = sandboxCallbackBridgeDirectories(queueDir);
const baseClient = createFileSystemSandboxCallbackBridgeQueueClient();
let listCalls = 0;
const client: SandboxCallbackBridgeQueueClient = {
...baseClient,
listJsonFiles: async (dirPath: string) => {
listCalls += 1;
if (listCalls <= 3) {
throw new Error("list requests failed: kex_exchange_identification: read: Connection reset by peer");
}
return baseClient.listJsonFiles(dirPath);
},
};
const seenPaths: string[] = [];
const worker = await startSandboxCallbackBridgeWorker({
client,
queueDir,
authorizeRequest: async () => null,
handleRequest: async (request) => {
seenPaths.push(request.path);
return {
status: 200,
headers: { "content-type": "application/json" },
body: JSON.stringify({ ok: true }),
};
},
});
cleanupFns.push(async () => {
await worker.stop();
});
const requestId = "transient-recovery-1";
await writeFile(
path.join(directories.requestsDir, `${requestId}.json`),
JSON.stringify({
id: requestId,
method: "GET",
path: "/api/agents/me",
query: "",
headers: {},
body: "",
}),
"utf8",
);
const responseFile = await waitForJsonFile(directories.responsesDir, 10_000);
const raw = await readFile(path.join(directories.responsesDir, responseFile), "utf8");
expect(JSON.parse(raw)).toMatchObject({ id: requestId, status: 200 });
expect(seenPaths).toEqual(["/api/agents/me"]);
expect(listCalls).toBeGreaterThan(3);
});
it("keeps the queue-directory setup on the startup step but resets the poll loop store", async () => {
// The worker starts inside the measured `bridge.paperclip` step. Its awaited
// queue-directory setup is startup work, so a `makeDir` `sandbox.exec` span
@@ -1510,7 +1573,7 @@ describe("sandbox callback bridge", () => {
})}\n`;
}
it("times out a stalled poll, writes a 503, and surfaces a run-level error", async () => {
it("times out a stalled poll, surfaces a run-level error, and recovers to deliver the request", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-hang-"));
cleanupDirs.push(rootDir);
@@ -1526,9 +1589,11 @@ describe("sandbox callback bridge", () => {
const client: SandboxCallbackBridgeQueueClient = {
...base,
// The first poll never resolves — a silently unresponsive sandbox channel.
// The per-iteration timeout must convert the hang into a caught error. The
// request never reaches the handler, so the recovery path can safely 503
// it. Later calls (the recovery's failPendingRequests) resolve.
// The per-iteration timeout must convert the hang into a caught error, and
// the loop must then back off and retry rather than die: the request never
// reached the handler, so the retry delivers the real response. A
// sustained outage is the watchdog's job (proven separately below), not a
// reason to fail a request one transient hang could still serve.
listJsonFiles: async (dir) => {
listCalls += 1;
if (listCalls === 1) {
@@ -1550,7 +1615,7 @@ describe("sandbox callback bridge", () => {
const responseFile = await waitForJsonFile(directories.responsesDir, 3_000);
const responseBody = await readFile(path.posix.join(directories.responsesDir, responseFile), "utf8");
expect(JSON.parse(responseBody).status).toBe(503);
expect(JSON.parse(responseBody).status).toBe(200);
expect(workerErrors.length).toBeGreaterThan(0);
expect(workerErrors[0]).toContain("timed out");
@@ -2261,10 +2326,13 @@ describe("sandbox callback bridge", () => {
});
it("retries a recovery 503 write that fails transiently, delivers the 503, and removes the request", async () => {
// The poll times out, so the recovery path aborts the queued request with a
// 503. The first 503 write fails, so the recovery must retry it inside the
// same pass. It then delivers the 503 and removes the request. The request is
// unclaimed, so its host mutation never ran; the 503 stays retry-safe.
// A request attempt times out (its first read hangs), so the loop's request
// catch runs the recovery pass, which aborts the queued request with a 503.
// The first 503 write fails, so the recovery must retry it inside the same
// pass. It then delivers the 503 and removes the request. The request is
// unclaimed, so its host mutation never ran; the 503 stays retry-safe. (A
// hung poll no longer triggers this pass — the loop backs off and retries
// the poll instead, and a sustained hang is the watchdog's job.)
const waitFor = async (predicate: () => boolean, timeoutMs: number) => {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
@@ -2286,27 +2354,26 @@ describe("sandbox callback bridge", () => {
requestBodies.set(requestPath, bridgeRequestJson("req-503-retry"));
const responseWrites: Array<{ path: string; status: number; body: string }> = [];
const requestRemovals: string[] = [];
let listCalls = 0;
let readCalls = 0;
let writeAttempts = 0;
const client: SandboxCallbackBridgeQueueClient = {
makeDir: async () => {},
makeDirs: async () => {},
// The first poll never resolves — a silently unresponsive sandbox channel.
// The per-iteration timeout converts the hang into a caught error, so the
// loop `catch` runs the recovery path. Later listings resolve, so the
// recovery enumerates and aborts the request.
listJsonFiles: async (dir) => {
if (dir !== directories.requestsDir) {
return [];
}
listCalls += 1;
if (listCalls === 1) {
return await new Promise<string[]>(() => {});
}
return [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort();
},
listJsonFiles: async (dir) =>
dir === directories.requestsDir
? [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort()
: [],
// The first read never resolves — a silently unresponsive sandbox channel
// hit mid-request, before the handler claim. The per-iteration timeout
// converts the hang into a caught error, so the loop's request catch runs
// the recovery pass. The recovery's own read resolves, so it can build and
// deliver the 503.
readTextFile: async (remotePath) => {
readCalls += 1;
if (readCalls === 1) {
return await new Promise<string>(() => {});
}
const body = requestBodies.get(remotePath);
if (body === undefined) {
throw new Error(`missing request ${remotePath}`);
@@ -2640,4 +2707,377 @@ describe("sandbox callback bridge", () => {
await worker.stop({ drainTimeoutMs: 50 });
});
async function prepareGatewayFixture(prefix: string) {
const rootDir = await mkdtemp(path.join(os.tmpdir(), prefix));
cleanupDirs.push(rootDir);
const localWorkspaceDir = path.join(rootDir, "local-workspace");
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
await mkdir(localWorkspaceDir, { recursive: true });
await mkdir(remoteWorkspaceDir, { recursive: true });
await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge test\n", "utf8");
const runner = createExecRunner();
const bridgeAsset = await createSandboxCallbackBridgeAsset();
cleanupFns.push(bridgeAsset.cleanup);
const prepared = await prepareCommandManagedRuntime({
runner,
spec: { remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000 },
adapterKey: "codex",
workspaceLocalDir: localWorkspaceDir,
assets: [{ key: "bridge", localDir: bridgeAsset.localDir }],
});
const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
return {
runner,
remoteWorkspaceDir,
assetRemoteDir: prepared.assetDirs.bridge,
queueDir,
directories: sandboxCallbackBridgeDirectories(queueDir),
bridgeToken: createSandboxCallbackBridgeToken(),
};
}
it("cleans up a timed-out request file and keeps serving after the host recovers", async () => {
const fixture = await prepareGatewayFixture("paperclip-bridge-timeout-clean-");
const bridge = await startSandboxCallbackBridgeServer({
runner: fixture.runner,
remoteCwd: fixture.remoteWorkspaceDir,
assetRemoteDir: fixture.assetRemoteDir,
queueDir: fixture.queueDir,
bridgeToken: fixture.bridgeToken,
timeoutMs: 30_000,
responseTimeoutMs: 600,
pollIntervalMs: 50,
});
cleanupFns.push(async () => {
await bridge.stop();
});
// No worker runs, so the request times out at the gateway.
const timedOut = await fetch(`${bridge.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${fixture.bridgeToken}` },
});
expect(timedOut.status).toBe(502);
await expect(timedOut.json()).resolves.toMatchObject({
error: expect.stringContaining("Timed out"),
});
// The gateway cleaned its own request file, so nothing counts toward the
// queue-depth cap after the caller gave up.
const leftover = (await readdir(fixture.directories.requestsDir).catch(() => [])).filter((name) =>
name.endsWith(".json"),
);
expect(leftover).toEqual([]);
// A worker that comes up afterwards serves the next request normally —
// the timeout neither wedged nor killed the gateway.
const worker = await startSandboxCallbackBridgeWorker({
client: createFileSystemSandboxCallbackBridgeQueueClient(),
queueDir: fixture.queueDir,
authorizeRequest: async () => null,
handleRequest: async () => ({
status: 200,
headers: { "content-type": "application/json" },
body: JSON.stringify({ ok: true }),
}),
});
cleanupFns.push(async () => {
await worker.stop();
});
const recovered = await fetch(`${bridge.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${fixture.bridgeToken}` },
});
expect(recovered.status).toBe(200);
await expect(recovered.json()).resolves.toMatchObject({ ok: true });
}, 30_000);
it("sweeps stale request files before rejecting at the queue-depth cap", async () => {
const fixture = await prepareGatewayFixture("paperclip-bridge-stale-sweep-");
const bridge = await startSandboxCallbackBridgeServer({
runner: fixture.runner,
remoteCwd: fixture.remoteWorkspaceDir,
assetRemoteDir: fixture.assetRemoteDir,
queueDir: fixture.queueDir,
bridgeToken: fixture.bridgeToken,
timeoutMs: 30_000,
responseTimeoutMs: 500,
pollIntervalMs: 50,
maxQueueDepth: 1,
});
cleanupFns.push(async () => {
await bridge.stop();
});
// Plant an orphaned request file (a killed caller, or a previous gateway
// process's leftover) and backdate it beyond the response deadline.
const orphanPath = path.join(fixture.directories.requestsDir, "orphan.json");
await writeFile(orphanPath, bridgeRequestJson("orphan"), "utf8");
const staleTime = new Date(Date.now() - 60_000);
await utimes(orphanPath, staleTime, staleTime);
// The queue sits at the cap, but the only entry is stale: the gateway must
// sweep it and admit the request instead of answering 503. With no worker
// the admitted request then times out (502) — proof it entered the queue.
const response = await fetch(`${bridge.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${fixture.bridgeToken}` },
});
expect(response.status).toBe(502);
const body = (await response.json()) as { error?: string };
expect(body.error ?? "").not.toContain("queue is full");
const leftover = (await readdir(fixture.directories.requestsDir).catch(() => [])).filter((name) =>
name.endsWith(".json"),
);
expect(leftover).toEqual([]);
}, 30_000);
it("skips a request file that vanished before the read instead of escalating", async () => {
// The gateway deletes a request file when its caller stops waiting. The
// worker's read then races the deletion; a vanished file must be a quiet
// skip, not a worker failure with a recovery pass.
const queueDir = "/virtual-bridge/vanished";
let listCalls = 0;
const handled: string[] = [];
const responseStatuses: number[] = [];
const { runtimeSpan, workerErrors } = createWorkerErrorCapture();
const client: SandboxCallbackBridgeQueueClient = {
makeDir: async () => {},
makeDirs: async () => {},
listJsonFiles: async () => {
listCalls += 1;
return listCalls === 1 ? ["ghost.json"] : [];
},
readTextFile: async () => {
throw new Error("cat: ghost.json: No such file or directory");
},
writeTextFile: async () => {},
writeResponseFile: async (_remotePath, body) => {
responseStatuses.push((JSON.parse(body.trim()) as { status: number }).status);
return { wrote: true };
},
rename: async () => {},
remove: async () => {},
};
const worker = await startSandboxCallbackBridgeWorker({
client,
queueDir,
runtimeSpan,
authorizeRequest: async () => null,
handleRequest: async (request) => {
handled.push(request.id);
return { status: 200, body: "ok" };
},
});
await new Promise((resolve) => setTimeout(resolve, 250));
await worker.stop({ drainTimeoutMs: 50 });
expect(handled).toEqual([]);
expect(responseStatuses).toEqual([]);
expect(workerErrors).toEqual([]);
});
it("embeds crash handlers and queue hygiene in the generated gateway source", () => {
// A crashed gateway is a dead loopback port for the rest of the run, so
// the generated source must keep its crash handlers and its stale-queue
// sweep. The readiness gate matters too: survival applies only after the
// gateway is adoptable, so a startup fault still fails fast. This pins
// their presence; the behavior is proven above and below.
const source = getSandboxCallbackBridgeServerSource();
expect(source).toContain('process.on("uncaughtException"');
expect(source).toContain('process.on("unhandledRejection"');
expect(source).toContain("gatewayReady");
expect(source).toContain("sweepStaleRequests");
});
it("exits fast when the gateway cannot bind its port instead of lingering un-ready", async () => {
// Before readiness, the crash handlers must not keep the process alive: a
// failed bind means the gateway can never serve, and surviving would only
// leave an un-ready zombie while the host waits out its readiness poll.
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-bind-fail-"));
cleanupDirs.push(rootDir);
const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
const queueDir = path.join(rootDir, "queue");
await mkdir(queueDir, { recursive: true });
const blocker = createServer();
await new Promise<void>((resolve) => {
blocker.listen(0, "127.0.0.1", resolve);
});
cleanupFns.push(
() =>
new Promise<void>((resolve) => {
blocker.close(() => resolve());
}),
);
const blockedPort = (blocker.address() as { port: number }).port;
const child = spawn(process.execPath, [entrypoint], {
env: {
...process.env,
PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir,
PAPERCLIP_BRIDGE_TOKEN: "test-token",
PAPERCLIP_BRIDGE_PORT: String(blockedPort),
},
stdio: ["ignore", "ignore", "pipe"],
});
let stderr = "";
child.stderr.on("data", (chunk) => {
stderr += chunk;
});
const exitCode = await new Promise<number | null>((resolve) => {
child.on("close", resolve);
});
expect(exitCode).toBe(1);
expect(stderr).toContain("[paperclip-bridge] server error");
expect(stderr).toContain("EADDRINUSE");
}, 15_000);
it("exits nonzero for the retired duplex_v1 mode instead of starting the queue gateway", async () => {
// The closed mode allowlist rejects `duplex_v1` before the queue-directory
// check, so a stale `duplex_v1` launch environment fails startup instead
// of silently falling through to the queue gateway.
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-mode-duplex-"));
cleanupDirs.push(rootDir);
const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
const queueDir = path.join(rootDir, "queue");
await mkdir(queueDir, { recursive: true });
const child = spawn(process.execPath, [entrypoint], {
env: {
...process.env,
PAPERCLIP_API_BRIDGE_MODE: "duplex_v1",
PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir,
PAPERCLIP_BRIDGE_TOKEN: "test-token",
PAPERCLIP_BRIDGE_PORT: "0",
},
stdio: ["ignore", "ignore", "pipe"],
});
let stderr = "";
child.stderr.on("data", (chunk) => {
stderr += chunk;
});
const exitCode = await new Promise<number | null>((resolve) => {
child.on("close", resolve);
});
expect(exitCode).not.toBe(0);
expect(stderr).toContain("Unsupported PAPERCLIP_API_BRIDGE_MODE: duplex_v1");
}, 15_000);
it("exits nonzero for an unknown bridge mode instead of starting the queue gateway", async () => {
// The closed mode allowlist rejects every value it does not name, not
// only the retired duplex transport.
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-mode-unknown-"));
cleanupDirs.push(rootDir);
const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
const queueDir = path.join(rootDir, "queue");
await mkdir(queueDir, { recursive: true });
const child = spawn(process.execPath, [entrypoint], {
env: {
...process.env,
PAPERCLIP_API_BRIDGE_MODE: "totally_unknown_mode",
PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir,
PAPERCLIP_BRIDGE_TOKEN: "test-token",
PAPERCLIP_BRIDGE_PORT: "0",
},
stdio: ["ignore", "ignore", "pipe"],
});
let stderr = "";
child.stderr.on("data", (chunk) => {
stderr += chunk;
});
const exitCode = await new Promise<number | null>((resolve) => {
child.on("close", resolve);
});
expect(exitCode).not.toBe(0);
expect(stderr).toContain("Unsupported PAPERCLIP_API_BRIDGE_MODE: totally_unknown_mode");
}, 15_000);
it("test_http2_gateway_writes_no_frame_between_ready_and_the_preface", async () => {
// Spawn the real generated gateway in http2_v1 mode and read its raw
// stdout bytes. The only frame-codec write on this path is the READY
// line; the very next bytes must be the HTTP/2 client connection preface
// with nothing in between, because the gateway hands stdout to the
// HTTP/2 client immediately after it writes READY and starts no
// heartbeat timer and writes no envelope frame on this path.
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-http2-gateway-"));
cleanupDirs.push(rootDir);
const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
const probe = createServer();
const assignedPort = await new Promise<number>((resolve, reject) => {
probe.once("error", reject);
probe.listen(0, "127.0.0.1", () => {
const address = probe.address();
if (!address || typeof address === "string") {
reject(new Error("Could not reserve a loopback port for the test."));
return;
}
probe.close(() => resolve(address.port));
});
});
const nonce = "test-nonce-http2";
const child = spawn(process.execPath, [entrypoint], {
env: {
...process.env,
PAPERCLIP_API_BRIDGE_MODE: "http2_v1",
PAPERCLIP_BRIDGE_TOKEN: "test-token",
PAPERCLIP_BRIDGE_PORT: String(assignedPort),
PAPERCLIP_BRIDGE_NONCE: nonce,
},
stdio: ["pipe", "pipe", "pipe"],
});
cleanupFns.push(async () => {
child.kill();
});
let stderr = "";
child.stderr.on("data", (chunk: Buffer) => {
stderr += chunk.toString("utf8");
});
const chunks: Buffer[] = [];
const preface = Buffer.from("505249202a20485454502f322e300d0a0d0a534d0d0a0d0a", "hex");
const firstBytes = await new Promise<Buffer>((resolve, reject) => {
const timer = setTimeout(
() => reject(new Error("Timed out waiting for the http2 gateway stdout. stderr: " + stderr)),
5000,
);
child.stdout.on("data", (chunk: Buffer) => {
chunks.push(chunk);
const total = Buffer.concat(chunks);
const newlineIndex = total.indexOf(0x0a);
if (newlineIndex !== -1 && total.length >= newlineIndex + 1 + preface.length) {
clearTimeout(timer);
resolve(total);
}
});
child.once("error", reject);
child.once("exit", (code) => {
clearTimeout(timer);
reject(new Error("The http2 gateway exited early with code " + String(code) + ". stderr: " + stderr));
});
});
const newlineIndex = firstBytes.indexOf(0x0a);
expect(newlineIndex).toBeGreaterThan(0);
const readyLine = firstBytes.subarray(0, newlineIndex).toString("utf8");
expect(JSON.parse(readyLine)).toMatchObject({ type: "ready", nonce });
const afterReady = firstBytes.subarray(newlineIndex + 1, newlineIndex + 1 + preface.length);
expect(afterReady).toEqual(preface);
}, 15_000);
});
@@ -1,7 +1,9 @@
import { createHash, randomBytes, randomUUID } from "node:crypto";
import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
import { promises as fs } from "node:fs";
import http2 from "node:http2";
import os from "node:os";
import path from "node:path";
import type { Duplex } from "node:stream";
import {
runWithoutActiveStep,
@@ -34,7 +36,8 @@ const DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES = 8 * 1024 * 1024;
// (PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS), so the host loop fails fast and writes
// 503 responses before the in-sandbox client gives up. A silently unresponsive
// sandbox channel makes a client call hang with no reject; this timeout turns
// that hang into a caught error, so the loop `catch` runs `failPendingRequests`.
// that hang into a caught error, so the poll loop can back off and retry while
// the watchdog below decides when to fail the queued requests.
const DEFAULT_BRIDGE_ITERATION_TIMEOUT_MS = 10_000;
// Watchdog backstop for a hang that the per-iteration timeout does not catch
// (for example many slow-but-under-timeout calls, or a stall outside the awaited
@@ -58,32 +61,27 @@ const MAX_BACKSTOP_WRITE_ATTEMPTS = 3;
// The delay between two 504 backstop write attempts. It is short, so all retries
// finish well under the in-sandbox 30s response deadline.
const BACKSTOP_WRITE_RETRY_MS = 50;
// Backoff cap between poll-loop retries after a transient iteration failure.
// The cap keeps a recovering loop probing often enough to resume before the
// in-sandbox 30s response deadline strands queued callers, while the
// exponential ramp below it keeps a hard-down channel from burning an exec
// call every poll interval.
const MAX_TRANSIENT_ITERATION_BACKOFF_MS = 5_000;
const REMOTE_WRITE_BASE64_CHUNK_SIZE = 32 * 1024;
export const SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT = "paperclip-bridge-server.mjs";
const SANDBOX_EXEC_CHANNEL_ENV = "PAPERCLIP_SANDBOX_EXEC_CHANNEL";
const SANDBOX_EXEC_CHANNEL_BRIDGE = "bridge";
// The two bridge modes the generated gateway supports. The file mode polls a
// request/response queue on disk. The duplex mode forwards one request frame to
// stdout and resolves one response frame from stdin. The generated `.mjs`
// selects the mode from `PAPERCLIP_API_BRIDGE_MODE`.
// The bridge modes the generated gateway supports. The file mode polls a
// request/response queue on disk. The http2 mode runs one Node HTTP/2 client
// session directly on stdin/stdout, after it sends the one READY line the
// host readiness gate expects. The generated `.mjs` selects the mode from
// `PAPERCLIP_API_BRIDGE_MODE`. The generated gateway rejects every other
// value with a fixed startup error, including the retired duplex transport.
// HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback.
const SANDBOX_CALLBACK_BRIDGE_FILE_MODE = "queue_v1";
export const SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE = "duplex_v1";
// The duplex gateway HTTP wait budget default. The gateway waits this long for a
// response frame before it answers the local caller with a 502 timeout. The
// value stays configurable through `PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS`, the
// same environment key the file mode reads.
const DEFAULT_DUPLEX_GATEWAY_WAIT_BUDGET_MS = 35_000;
// How often the duplex gateway writes a heartbeat frame to stdout.
const DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS = 5_000;
// The duplex gateway treats the channel as lost when no inbound frame arrives on
// stdin within this window.
const DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_TIMEOUT_MS = 20_000;
// After a loss, the duplex gateway answers each new local request with a 503,
// then exits when this grace ends. The grace gives the local caller time to read
// the 409 for an outstanding request and a 503 for a new request.
const DEFAULT_DUPLEX_GATEWAY_LOSS_EXIT_GRACE_MS = 1_000;
/** The active non-file transport mode. */
export const SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE = "http2_v1";
/** Span name that wraps one Paperclip-API callback request — read the request,
* write the response, and remove the request file. */
@@ -976,7 +974,22 @@ export async function startSandboxCallbackBridgeWorker(input: {
await writeAbortedHandlerBackstop(fileName, guard, lastWriteError);
};
try {
const raw = await input.client.readTextFile(requestPath);
let raw: string;
try {
raw = await input.client.readTextFile(requestPath);
} catch (error) {
// The gateway deletes a request file when its caller stops waiting
// (client-side timeout cleanup). A read that fails because the file is
// gone is that benign race, not a channel fault: confirm the file
// vanished and skip quietly instead of escalating into a recovery
// pass. A file that is still listed rethrows, so a real read fault
// keeps its existing handling.
const remaining = await input.client.listJsonFiles(directories.requestsDir).catch(() => null);
if (remaining !== null && !remaining.includes(fileName)) {
return;
}
throw error;
}
let request: SandboxCallbackBridgeRequest;
try {
request = JSON.parse(raw) as SandboxCallbackBridgeRequest;
@@ -1380,13 +1393,54 @@ export async function startSandboxCallbackBridgeWorker(input: {
watchdogTimer.unref();
}
try {
// Consecutive transient poll failures. A single failed list call — one
// reset or slow sandbox exec — must not end the relay for the rest of the
// run: the in-sandbox gateway keeps queueing requests, so a dead loop
// strands every later API call from the agent (its status writes then look
// like connection failures and the issue loses its disposition). Back off
// and retry the poll instead. The watchdog stays the escalation path for a
// sustained outage — it fires after `watchdogTimeoutMs` without a
// successful iteration and fails the queued requests fast, while this loop
// keeps probing for recovery.
let consecutivePollFailures = 0;
while (true) {
const fileNames = await withTimeout(
input.client.listJsonFiles(directories.requestsDir),
iterationTimeoutMs,
"Sandbox callback bridge list requests",
);
if (fileNames.length === 0) {
let fileNames: string[];
try {
fileNames = await withTimeout(
input.client.listJsonFiles(directories.requestsDir),
iterationTimeoutMs,
"Sandbox callback bridge list requests",
);
consecutivePollFailures = 0;
} catch (error) {
if (stopping) {
break;
}
consecutivePollFailures += 1;
const message = `${buildWorkerFailureMessage(error)} (transient poll failure ${consecutivePollFailures}; retrying)`;
if (consecutivePollFailures === 1) {
// Put the first failure of a streak on the run trace; later repeats
// only warn, so a flapping channel does not spam failed spans.
await surfaceRunError(new Error(message));
} else {
console.warn(`[paperclip] ${message}`);
}
const backoffMs = Math.min(
pollIntervalMs * 2 ** consecutivePollFailures,
MAX_TRANSIENT_ITERATION_BACKOFF_MS,
);
await new Promise((resolve) => setTimeout(resolve, backoffMs));
continue;
}
// A file whose attempt is still in flight (or waiting on its 504
// backstop) is not actionable: `processRequestFile` would skip it via
// the guard map. Treat an all-guarded listing like an empty one and
// sleep a poll interval. Re-listing immediately would spin the loop —
// an exec storm against a real sandbox channel, and with an in-memory
// client a pure-microtask loop that starves every timer in the process
// (including the guard's own backstop and abort timers).
const actionableFileNames = fileNames.filter((fileName) => !inFlightRequestGuards.has(fileName));
if (actionableFileNames.length === 0) {
lastSuccessfulIterationAt = Date.now();
if (stopping) {
break;
@@ -1394,7 +1448,7 @@ export async function startSandboxCallbackBridgeWorker(input: {
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
continue;
}
for (const fileName of fileNames) {
for (const fileName of actionableFileNames) {
if (stopping && Date.now() >= stopDeadline) break;
inFlight += 1;
try {
@@ -1406,7 +1460,7 @@ export async function startSandboxCallbackBridgeWorker(input: {
// `task.run` after it. Without a runner, the request runs under the
// run parent with no wrapper span, exactly like the earlier behavior.
// The per-iteration timeout wraps the whole request, so a hung
// request rejects and the loop `catch` runs `failPendingRequests`.
// request rejects and the catch below runs the recovery pass.
await withTimeout(
input.runtimeSpan
? input.runtimeSpan(CALLBACK_BRIDGE_RELAY_REQUEST_SPAN, () =>
@@ -1419,6 +1473,23 @@ export async function startSandboxCallbackBridgeWorker(input: {
`Sandbox callback bridge process request ${fileName}`,
);
lastSuccessfulIterationAt = Date.now();
} catch (error) {
// A single request attempt failed or hung. Run the same recovery
// pass the loop previously died on — abort the in-flight handler
// (its 504 backstop keeps the caller from stranding) and 503 the
// unclaimed queued requests — but keep the loop alive afterward. A
// caller that sees the retry-safe 503 re-queues, and the recovered
// loop serves the retry; the old terminal catch left every later
// request to strand instead.
const message = buildWorkerFailureMessage(error);
await surfaceRunError(new Error(message));
try {
await failPendingRequests(message, { abandonInFlight: true });
} catch (failPendingError) {
console.warn(
`[paperclip] sandbox callback bridge failed to abort queued requests after a request failure: ${failPendingError instanceof Error ? failPendingError.message : String(failPendingError)}`,
);
}
} finally {
inFlight -= 1;
}
@@ -1752,6 +1823,208 @@ export async function startSandboxCallbackBridgeServer(input: {
};
}
// ---------------------------------------------------------------------------
// Sandbox HTTP/2 client gateway
//
// This gateway turns each local loopback request into one HTTP/2 stream to
// the host server in `http2-bridge-server.ts`. It sits beside the file-mode
// gateway above; it changes neither of them. The generated in-sandbox
// entrypoint and the transport-selection path already select it.
// ---------------------------------------------------------------------------
/**
* Constant-time bridge-token compare. Both this sandbox gateway and the host
* server in `http2-bridge-server.ts` import this one helper, so the gateway
* check and the independent host check (accepted security fix 4) apply the
* exact same comparison rule. A length mismatch returns `false` without a
* `timingSafeEqual` call, because `timingSafeEqual` throws on unequal buffer
* lengths; both operands are bridge tokens of near-fixed length, so this one
* length branch leaks no useful timing signal.
*/
export function compareBridgeTokensConstantTime(
expected: string,
received: string | null | undefined,
): boolean {
const expectedBytes = Buffer.from(expected, "utf8");
const receivedBytes = Buffer.from(typeof received === "string" ? received : "", "utf8");
if (expectedBytes.length !== receivedBytes.length) return false;
return timingSafeEqual(expectedBytes, receivedBytes);
}
const SANDBOX_HTTP2_GATEWAY_DEFAULT_AUTHORITY = "bridge.internal";
/** One local request the gateway forwards as one HTTP/2 stream. */
export interface SandboxHttp2BridgeGatewayRequest {
method: string;
path: string;
query: string;
headers: Record<string, string>;
body: Buffer;
/**
* The token the local caller presented. The gateway check (accepted
* security fix 4 keeps this alongside the independent host check) compares
* it against the per-run bridge token before it opens a stream.
*/
receivedToken: string | null | undefined;
}
/** The response one forwarded HTTP/2 stream carried back. */
export interface SandboxHttp2BridgeGatewayResponse {
status: number;
headers: Record<string, string>;
body: Buffer;
}
export interface SandboxHttp2BridgeGateway {
/** Forward one local request as one HTTP/2 stream over the client session. */
forwardRequest(
request: SandboxHttp2BridgeGatewayRequest,
): Promise<SandboxHttp2BridgeGatewayResponse>;
/** Close the HTTP/2 client session. Safe to call more than one time. */
close(): Promise<void>;
}
export interface CreateSandboxHttp2BridgeGatewayOptions {
/**
* The per-run bridge token. The gateway checks every request against it,
* then attaches it to the outbound HTTP/2 stream as the `authorization`
* header, so the host can run its own independent check.
*/
bridgeToken: string;
/**
* Open the transport the HTTP/2 client session runs on. Returns a `Duplex`
* already connected to the host — the sandbox process's own channel in
* production, or one side of a paired in-memory `Duplex` in a test.
*/
createConnection: () => Duplex;
/** The `:authority` pseudo-header value. The channel carries no real network
* address, so this is a fixed label. The default is `bridge.internal`. */
authority?: string;
/** The header allowlist applied to every outbound request. The default is
* {@link DEFAULT_SANDBOX_CALLBACK_BRIDGE_HEADER_ALLOWLIST}. */
headerAllowlist?: readonly string[];
/**
* The sink for a GOAWAY the host sends. The host names the last client
* stream ID it processed; a caller classifies each of its own dispatched
* stream IDs against it with `classifyStreamAgainstGoaway` in
* `http2-bridge-server.ts` to know which requests need a retry elsewhere.
*/
onGoaway?: (record: { lastStreamId: number; errorCode: number }) => void;
}
function forwardOneHttp2Request(
session: http2.ClientHttp2Session,
request: {
bridgeToken: string;
method: string;
path: string;
query: string;
headers: Record<string, string>;
body: Buffer;
},
): Promise<SandboxHttp2BridgeGatewayResponse> {
return new Promise((resolve, reject) => {
const query = request.query.trim();
const pathWithQuery =
query.length === 0 ? request.path : `${request.path}${query.startsWith("?") ? query : `?${query}`}`;
const requestHeaders: http2.OutgoingHttpHeaders = {
":method": request.method,
":path": pathWithQuery,
authorization: `Bearer ${request.bridgeToken}`,
...request.headers,
};
let stream: http2.ClientHttp2Stream;
try {
stream = session.request(requestHeaders, { endStream: request.body.length === 0 });
} catch (error) {
reject(error instanceof Error ? error : new Error(String(error)));
return;
}
const chunks: Buffer[] = [];
let responseHeaders: Record<string, string> = {};
let status = 502;
let settled = false;
const settle = (run: () => void) => {
if (settled) return;
settled = true;
run();
};
stream.on("response", (headers) => {
const rawStatus = headers[":status"];
status = typeof rawStatus === "number" ? rawStatus : Number(rawStatus) || 502;
responseHeaders = {};
for (const [key, value] of Object.entries(headers)) {
if (key.startsWith(":") || value == null) continue;
responseHeaders[key] = Array.isArray(value) ? value.join(", ") : String(value);
}
});
stream.on("data", (chunk: Buffer) => chunks.push(chunk));
stream.once("end", () => settle(() => resolve({ status, headers: responseHeaders, body: Buffer.concat(chunks) })));
stream.once("error", (error) =>
settle(() => reject(error instanceof Error ? error : new Error(String(error)))),
);
stream.once("aborted", () => settle(() => reject(new Error("Bridge HTTP/2 stream aborted."))));
if (request.body.length > 0) {
stream.end(request.body);
} else if (!stream.writableEnded) {
stream.end();
}
});
}
/**
* Create the sandbox HTTP/2 client gateway. It opens one HTTP/2 client
* session on the transport `createConnection` returns, and forwards each
* local request the caller hands it (already checked against the bridge
* token — see {@link SandboxHttp2BridgeGatewayRequest.receivedToken}) as one
* HTTP/2 stream. It keeps the header allowlist on the sandbox side, exactly
* as the file-mode gateway does.
*/
export function createSandboxHttp2BridgeGateway(
options: CreateSandboxHttp2BridgeGatewayOptions,
): SandboxHttp2BridgeGateway {
const authority = options.authority?.trim() || SANDBOX_HTTP2_GATEWAY_DEFAULT_AUTHORITY;
const headerAllowlist = options.headerAllowlist ?? DEFAULT_SANDBOX_CALLBACK_BRIDGE_HEADER_ALLOWLIST;
const session = http2.connect(`http://${authority}`, {
createConnection: options.createConnection,
});
// A session-level fault fails every in-flight `forwardRequest` call through
// that stream's own `error`/`aborted` handler. This listener only stops
// Node from raising an unhandled `error` event for the session itself.
session.on("error", () => undefined);
session.on("goaway", (errorCode: number, lastStreamId: number) => {
options.onGoaway?.({ lastStreamId, errorCode });
});
return {
forwardRequest(request: SandboxHttp2BridgeGatewayRequest): Promise<SandboxHttp2BridgeGatewayResponse> {
// The gateway check (accepted security fix 4 keeps this as well as the
// independent host-side check): a request whose token does not match
// the per-run bridge token never opens a stream.
if (!compareBridgeTokensConstantTime(options.bridgeToken, request.receivedToken)) {
return Promise.reject(new Error("Invalid bridge token."));
}
return forwardOneHttp2Request(session, {
bridgeToken: options.bridgeToken,
method: request.method,
path: request.path,
query: request.query,
headers: sanitizeSandboxCallbackBridgeHeaders(request.headers, headerAllowlist),
body: request.body,
});
},
close(): Promise<void> {
return new Promise((resolve) => {
if (session.closed || session.destroyed) {
resolve();
return;
}
session.close(() => resolve());
});
},
};
}
/**
* The zero-dependency codec the generated duplex gateway embeds. It is a plain
* JavaScript copy of the host codec in `duplex-frame-codec.ts`. It uses only the
@@ -1763,7 +2036,8 @@ export async function startSandboxCallbackBridgeServer(input: {
* compatible with the host codec. {@link getSandboxDuplexGatewayCodecSource}
* returns this exact source so a test can run every fixture vector against it.
*/
const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 1;
const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 2;
const DUPLEX_BODY_CHUNK_RAW_BYTES = 256 * 1024;
const DEFAULT_MAX_DUPLEX_FRAME_BYTES = 1000000;
const DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES = 256;
const DEFAULT_MAX_DUPLEX_DECODER_BYTES = ${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES};
@@ -1792,6 +2066,10 @@ function duplexIsStringRecord(value) {
return true;
}
function duplexIsSafeNonNegativeInteger(value) {
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
}
function encodeDuplexFrame(frame) {
return JSON.stringify(frame) + "\\n";
}
@@ -1828,6 +2106,8 @@ function duplexValidateFrame(frame) {
return duplexValidateRequest(frame);
case "response":
return duplexValidateResponse(frame);
case "body_chunk":
return duplexValidateBodyChunk(frame);
case "ready":
return duplexValidateReady(frame);
case "heartbeat":
@@ -1847,7 +2127,7 @@ function duplexValidateRequest(frame) {
typeof frame.method !== "string" ||
typeof frame.path !== "string" ||
typeof frame.query !== "string" ||
typeof frame.body !== "string" ||
!duplexIsSafeNonNegativeInteger(frame.bodyByteCount) ||
!duplexIsStringRecord(frame.headers)
) {
return duplexFail("malformed_frame", "request frame has a missing or wrong-typed field");
@@ -1862,7 +2142,7 @@ function duplexValidateResponse(frame) {
if (
typeof frame.id !== "string" ||
typeof frame.status !== "number" ||
typeof frame.body !== "string" ||
!duplexIsSafeNonNegativeInteger(frame.bodyByteCount) ||
!duplexIsStringRecord(frame.headers) ||
typeof frame.outcome !== "string" ||
!DUPLEX_RESPONSE_OUTCOMES.has(frame.outcome)
@@ -1875,6 +2155,19 @@ function duplexValidateResponse(frame) {
return duplexOk(frame);
}
function duplexValidateBodyChunk(frame) {
if (typeof frame.id !== "string" || typeof frame.data !== "string") {
return duplexFail("malformed_frame", "body_chunk frame has a missing or wrong-typed field");
}
if (!duplexIsSafeNonNegativeInteger(frame.seq)) {
return duplexFail("malformed_frame", "body_chunk frame has a missing or wrong-typed seq");
}
if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) {
return duplexFail("id_too_large", "body_chunk frame id exceeds the maximum size");
}
return duplexOk(frame);
}
function duplexValidateReady(frame) {
if (typeof frame.nonce !== "string") {
return duplexFail("malformed_frame", "ready frame has a missing or wrong-typed nonce");
@@ -1984,6 +2277,8 @@ export function getSandboxCallbackBridgeServerSource(): string {
import { createServer } from "node:http";
import { promises as fs } from "node:fs";
import path from "node:path";
import http2 from "node:http2";
import { Duplex } from "node:stream";
const bridgeMode = process.env.PAPERCLIP_API_BRIDGE_MODE || "${SANDBOX_CALLBACK_BRIDGE_FILE_MODE}";
const queueDir = process.env.PAPERCLIP_BRIDGE_QUEUE_DIR;
@@ -1991,36 +2286,18 @@ const bridgeToken = process.env.PAPERCLIP_BRIDGE_TOKEN;
const host = process.env.PAPERCLIP_BRIDGE_HOST || "127.0.0.1";
const port = Number(process.env.PAPERCLIP_BRIDGE_PORT || "0");
// The host assigns the loopback port and passes it through the launch
// environment. The duplex gateway binds exactly this port; it never selects a
// environment. The gateway binds exactly this port; it never selects a
// different one. The host also passes one random per-open nonce here. The
// gateway echoes it in the READY frame so the host correlates READY with this
// channel open. The nonce is a liveness signal, not authentication.
const bridgeNonce = process.env.PAPERCLIP_BRIDGE_NONCE || "";
const pollIntervalMs = Number(process.env.PAPERCLIP_BRIDGE_POLL_INTERVAL_MS || "100");
const responseTimeoutMs = Number(
process.env.PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS ||
(bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}"
? "${DEFAULT_DUPLEX_GATEWAY_WAIT_BUDGET_MS}"
: "${DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS}"),
process.env.PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS || "${DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS}",
);
const maxQueueDepth = Number(process.env.PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH || "${DEFAULT_BRIDGE_MAX_QUEUE_DEPTH}");
const maxBodyBytes = Number(process.env.PAPERCLIP_BRIDGE_MAX_BODY_BYTES || "${DEFAULT_BRIDGE_MAX_BODY_BYTES}");
// The host passes the separate sandbox-process raw-decoder cap here. The in-sandbox
// decoder enforces it locally under the "sandbox_process" scope; it never shares the
// host aggregate byte ledger.
const maxDuplexDecoderBytes = Number(
process.env.PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES || "${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}",
);
const heartbeatIntervalMs = Number(
process.env.PAPERCLIP_BRIDGE_HEARTBEAT_INTERVAL_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS}",
);
const heartbeatTimeoutMs = Number(
process.env.PAPERCLIP_BRIDGE_HEARTBEAT_TIMEOUT_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_TIMEOUT_MS}",
);
const lossExitGraceMs = Number(
process.env.PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS || "${DEFAULT_DUPLEX_GATEWAY_LOSS_EXIT_GRACE_MS}",
);
// The header allowlist. Both the file gateway and the duplex gateway strip an
// The header allowlist. Both the file gateway and the http2 gateway strip an
// inbound request to these headers before they forward it. One copy serves both
// modes. The route allowlist stays on the host: both modes forward a request to
// the host, and the host enforces the same route allowlist for each.
@@ -2029,10 +2306,48 @@ const allowedHeaders = new Set(${JSON.stringify([...DEFAULT_SANDBOX_CALLBACK_BRI
if (!bridgeToken) {
throw new Error("PAPERCLIP_BRIDGE_TOKEN is required.");
}
if (bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}" && !queueDir) {
// Closed allowlist for the bridge mode. The generated gateway supports exactly
// two transports: http2 and the file-mode queue. Every other value, including
// the retired duplex transport, fails startup at once instead of falling
// through to a mode that never ran. This check runs before the queue-directory
// check below, so an unsupported mode never reaches a state where a missing
// queue directory masks the real problem.
if (
bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}" &&
bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_FILE_MODE}"
) {
throw new Error("Unsupported PAPERCLIP_API_BRIDGE_MODE: " + bridgeMode);
}
if (bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}" && !queueDir) {
throw new Error("PAPERCLIP_BRIDGE_QUEUE_DIR and PAPERCLIP_BRIDGE_TOKEN are required.");
}
// A crashed gateway is a dead loopback port for the rest of the run: nothing
// inside the sandbox respawns this process, and every later agent API call
// then fails at the connection level. Once the gateway is ready, log an
// uncaught fault to stderr (the host redirects it into logs/bridge.log) and
// keep serving — the relay holds no state a fault can corrupt beyond the one
// request it interrupted. Before readiness the same fault means the gateway
// can never become usable (a failed bind, a failed readiness write), so exit
// instead: surviving there only leaves an un-ready zombie behind while the
// host waits out its readiness poll.
let gatewayReady = false;
process.on("uncaughtException", (error) => {
process.stderr.write(
"[paperclip-bridge] uncaught exception: " + (error && error.stack ? error.stack : String(error)) + "\\n",
);
if (!gatewayReady) {
process.exit(1);
}
});
process.on("unhandledRejection", (reason) => {
const detail = reason && typeof reason === "object" && "stack" in reason ? reason.stack : String(reason);
process.stderr.write("[paperclip-bridge] unhandled rejection: " + detail + "\\n");
if (!gatewayReady) {
process.exit(1);
}
});
// The embedded zero-dependency frame codec. The duplex gateway uses it; the file
// gateway ignores it.
${DUPLEX_GATEWAY_CODEC_SOURCE}
@@ -2092,6 +2407,24 @@ async function runFileGateway() {
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".json")).length;
}
// Delete request files older than the response deadline. Every live caller
// cleans its own request file when it times out, so a file this old is an
// orphan: its writer was killed mid-wait, or a previous gateway process died
// and left its queue behind. Orphans otherwise count toward the queue-depth
// cap forever and wedge the gateway at a permanent 503.
async function sweepStaleRequests() {
const staleBefore = Date.now() - responseTimeoutMs - 2000;
const entries = await fs.readdir(requestsDir, { withFileTypes: true }).catch(() => []);
for (const entry of entries) {
if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
const filePath = path.posix.join(requestsDir, entry.name);
const stats = await fs.stat(filePath).catch(() => null);
if (stats && stats.mtimeMs < staleBefore) {
await fs.rm(filePath, { force: true }).catch(() => undefined);
}
}
}
async function waitForResponse(requestId) {
const responsePath = path.posix.join(responsesDir, \`\${requestId}.json\`);
const deadline = Date.now() + responseTimeoutMs;
@@ -2116,8 +2449,13 @@ async function runFileGateway() {
}
if (await queueDepth() >= maxQueueDepth) {
writeJsonResponse(res, 503, { error: "Bridge request queue is full." });
return;
// Reclaim orphaned request files before rejecting; only a queue that
// is genuinely full of live requests gets the 503.
await sweepStaleRequests();
if (await queueDepth() >= maxQueueDepth) {
writeJsonResponse(res, 503, { error: "Bridge request queue is full." });
return;
}
}
const url = new URL(req.url || "/", "http://127.0.0.1");
@@ -2142,7 +2480,19 @@ async function runFileGateway() {
await fs.writeFile(tempPath, \`\${JSON.stringify(payload)}\\n\`, "utf8");
await fs.rename(tempPath, requestPath);
const response = await waitForResponse(requestId);
let response;
try {
response = await waitForResponse(requestId);
} catch (error) {
// The host never delivered a response inside the deadline. Remove this
// request's file so it cannot pile up toward the queue-depth cap. The
// host's normal response write is guarded on the request file, so the
// removal also tells the host that no caller waits anymore. Without
// this cleanup a stalled host wedges the gateway at the cap and every
// later request gets an immediate 503 until run end.
await fs.rm(requestPath, { force: true }).catch(() => undefined);
throw error;
}
const responseHeaders = response.headers || {};
// The host marks a possibly-committed mutation with an indeterminate outcome.
// The host cannot cancel a host operation that is in flight, so the mutation
@@ -2180,7 +2530,24 @@ async function runFileGateway() {
await fs.mkdir(responsesDir, { recursive: true });
await fs.mkdir(logsDir, { recursive: true });
// Newer Node runtimes do not reliably surface a failed bind through
// uncaughtException here: with nothing else keeping the event loop alive,
// the process can drain and exit 0 before the error event is delivered
// (observed on Node 24/25; Node 22 delivered it). Attach an explicit error
// listener and pin the loop with a keepalive until the bind settles, so a
// startup failure exits 1 with the fault on stderr on every runtime.
const bindKeepalive = setInterval(() => {}, 1000);
server.once("error", (error) => {
clearInterval(bindKeepalive);
process.stderr.write(
"[paperclip-bridge] server error: " + (error && error.stack ? error.stack : String(error)) + "\\n",
);
if (!gatewayReady) {
process.exit(1);
}
});
server.listen(port, host, async () => {
clearInterval(bindKeepalive);
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("Bridge server did not expose a TCP address.");
@@ -2195,107 +2562,136 @@ async function runFileGateway() {
const tempReadyFile = \`\${readyFile}.tmp\`;
await fs.writeFile(tempReadyFile, JSON.stringify(ready), "utf8");
await fs.rename(tempReadyFile, readyFile);
// The readiness file is on disk, so the host will adopt this process.
// From here on an uncaught fault must not kill the listener.
gatewayReady = true;
});
}
function runDuplexGateway() {
// One outstanding local request per id. Each entry holds the HTTP resolver and
// the wait-budget timer.
const pending = new Map();
let unavailable = false;
let lossTriggered = false;
let lastInboundAt = Date.now();
// ---------------------------------------------------------------------------
// http2_v1: run one Node HTTP/2 client session directly on stdin/stdout.
//
// This gateway writes exactly one frame-codec line before it hands stdin and
// stdout to the HTTP/2 client: the READY line, so the host readiness gate
// accepts the same handshake it already accepts for every mode. After that
// one write, only the HTTP/2 client touches stdout: this function calls
// writeFrame no more, and it starts no heartbeat timer, so no non-HTTP/2
// writer can put a byte on stdout between the READY line and the client
// connection preface.
// ---------------------------------------------------------------------------
function createStdioDuplex() {
const duplex = new Duplex({
read() {
// process.stdin pushes bytes through the "data" listener below; there
// is nothing to pull on demand here.
},
write(chunk, _encoding, callback) {
const flushed = process.stdout.write(chunk);
if (flushed) {
callback();
} else {
process.stdout.once("drain", () => callback());
}
},
});
process.stdin.on("data", (chunk) => {
duplex.push(chunk);
});
process.stdin.on("end", () => {
duplex.push(null);
});
process.stdin.on("error", (error) => {
duplex.destroy(error instanceof Error ? error : new Error(String(error)));
});
return duplex;
}
function runHttp2Gateway() {
function diag(message) {
// Diagnostics go to stderr only. Stdout carries frames.
// Diagnostics go to stderr only, the same as every other mode.
process.stderr.write("[paperclip-bridge] " + message + "\\n");
}
function writeFrame(frame) {
process.stdout.write(encodeDuplexFrame(frame));
}
function stopTimers() {
clearInterval(heartbeatSendTimer);
clearInterval(heartbeatWatchTimer);
const authority = "bridge.internal";
let session = null;
let unavailable = false;
function openSession() {
if (session) return session;
const stdio = createStdioDuplex();
session = http2.connect("http://" + authority, {
createConnection: () => stdio,
});
session.on("error", () => {
unavailable = true;
});
session.on("close", () => {
unavailable = true;
});
session.on("goaway", (errorCode, lastStreamId) => {
diag("host sent GOAWAY (errorCode=" + errorCode + ", lastStreamId=" + lastStreamId + ")");
});
return session;
}
// Loss behavior. On stdin end, a close frame, or a heartbeat timeout, answer
// every outstanding request with a non-retryable 409 (the request may have
// reached the host), answer every new request with a retryable 503 (the
// request never left the gateway), then exit. The gateway never replays a
// request.
function triggerLoss(reason) {
if (lossTriggered) return;
lossTriggered = true;
unavailable = true;
diag("duplex channel lost: " + reason);
stopTimers();
for (const entry of pending.values()) {
clearTimeout(entry.timer);
entry.resolve({
status: 409,
headers: {
"content-type": "application/json",
"x-paperclip-bridge-outcome": "indeterminate",
function forwardOverHttp2(request) {
return new Promise((resolve, reject) => {
const activeSession = openSession();
const query = request.query || "";
const pathWithQuery =
query.length === 0 ? request.path : request.path + (query.charAt(0) === "?" ? query : "?" + query);
const outboundHeaders = Object.assign(
{
":method": request.method,
":path": pathWithQuery,
authorization: "Bearer " + bridgeToken,
},
body: JSON.stringify({ error: "outcome_indeterminate" }),
});
}
pending.clear();
const exitTimer = setTimeout(() => process.exit(0), lossExitGraceMs);
if (typeof exitTimer.unref === "function") exitTimer.unref();
}
function handleInboundFrame(frame) {
if (frame.type === "response") {
const entry = pending.get(frame.id);
if (!entry) return;
pending.delete(frame.id);
clearTimeout(entry.timer);
// Map an indeterminate outcome to a non-retryable 409, the same contract
// the file gateway applies through the outcome header.
const statusCode =
frame.outcome === "indeterminate" ? 409 : typeof frame.status === "number" ? frame.status : 200;
entry.resolve({
status: statusCode,
headers: frame.headers || {},
body: typeof frame.body === "string" ? frame.body : "",
});
return;
}
if (frame.type === "close") {
triggerLoss("host sent a close frame");
return;
}
// A heartbeat, ready, error, or request frame proves liveness but needs no
// local action here.
}
const decoder = new DuplexFrameDecoder({ maxAggregateBytes: maxDuplexDecoderBytes });
process.stdin.on("data", (chunk) => {
lastInboundAt = Date.now();
for (const result of decoder.push(chunk)) {
if (!result.ok) {
diag("dropped an inbound frame: " + result.error.code);
continue;
request.headers,
);
let stream;
try {
stream = activeSession.request(outboundHeaders, { endStream: request.body.length === 0 });
} catch (error) {
reject(error instanceof Error ? error : new Error(String(error)));
return;
}
handleInboundFrame(result.frame);
}
});
process.stdin.on("end", () => triggerLoss("stdin reached end of file"));
process.stdin.on("error", (error) =>
triggerLoss("stdin error: " + (error && error.message ? error.message : String(error))),
);
const heartbeatSendTimer = setInterval(() => {
writeFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" });
}, heartbeatIntervalMs);
const heartbeatWatchTimer = setInterval(() => {
if (Date.now() - lastInboundAt > heartbeatTimeoutMs) {
triggerLoss("no inbound frame within the heartbeat timeout");
}
}, Math.max(200, Math.floor(heartbeatTimeoutMs / 4)));
const chunks = [];
let status = 502;
let responseHeaders = {};
let settled = false;
const settle = (run) => {
if (settled) return;
settled = true;
run();
};
stream.on("response", (headers) => {
const raw = headers[":status"];
status = typeof raw === "number" ? raw : Number(raw) || 502;
responseHeaders = {};
for (const [key, value] of Object.entries(headers)) {
if (key.charAt(0) === ":" || value == null) continue;
responseHeaders[key] = Array.isArray(value) ? value.join(", ") : String(value);
}
});
stream.on("data", (chunk) => chunks.push(chunk));
stream.once("end", () =>
settle(() => resolve({ status: status, headers: responseHeaders, body: Buffer.concat(chunks) })),
);
stream.once("error", (error) =>
settle(() => reject(error instanceof Error ? error : new Error(String(error)))),
);
stream.once("aborted", () => settle(() => reject(new Error("Bridge HTTP/2 stream aborted."))));
if (request.body.length > 0) {
stream.end(request.body);
} else if (!stream.writableEnded) {
stream.end();
}
});
}
const server = createServer(async (req, res) => {
try {
@@ -2309,61 +2705,32 @@ function runDuplexGateway() {
writeJsonResponse(res, 503, { error: "bridge_unavailable" });
return;
}
if (pending.size >= maxQueueDepth) {
writeJsonResponse(res, 503, { error: "Bridge request queue is full." });
return;
}
const url = new URL(req.url || "/", "http://127.0.0.1");
const contentType = typeof req.headers["content-type"] === "string" ? req.headers["content-type"] : "";
if (req.method && req.method !== "GET" && req.method !== "HEAD" && !/json/i.test(contentType)) {
writeJsonResponse(res, 415, { error: "Bridge only accepts JSON request bodies." });
return;
}
const requestId = randomUUID();
const requestBody = await readBody(req);
if (unavailable) {
writeJsonResponse(res, 503, { error: "bridge_unavailable" });
const requestBodyBuffer = Buffer.from(await readBody(req), "utf8");
let response;
try {
response = await forwardOverHttp2({
method: req.method || "GET",
path: url.pathname,
query: url.search,
headers: normalizeHeaders(req.headers),
body: requestBodyBuffer,
});
} catch (error) {
writeJsonResponse(res, 502, { error: error instanceof Error ? error.message : String(error) });
return;
}
const requestFrame = {
version: DUPLEX_FRAME_VERSION,
type: "request",
id: requestId,
method: req.method || "GET",
path: url.pathname,
query: url.search,
headers: normalizeHeaders(req.headers),
body: requestBody,
};
// Enforce the frame size bound on encode. When the request body makes the
// frame exceed the bound, fail this one local request with a clean 413. Do
// not write the frame and do not trigger loss. The frame never leaves the
// gateway, so no other in-flight request is affected and the channel stays
// open.
const encodedRequest = encodeDuplexFrameChecked(requestFrame);
if (!encodedRequest.ok) {
writeJsonResponse(res, 413, { error: "request_too_large" });
return;
}
const response = await new Promise((resolve) => {
const timer = setTimeout(() => {
if (pending.delete(requestId)) {
resolve({
status: 502,
headers: { "content-type": "application/json" },
body: JSON.stringify({ error: "Timed out waiting for host bridge response." }),
});
}
}, responseTimeoutMs);
pending.set(requestId, { resolve: resolve, timer: timer });
process.stdout.write(encodedRequest.line);
});
res.statusCode = typeof response.status === "number" ? response.status : 200;
for (const [key, value] of Object.entries(response.headers || {})) {
if (typeof value !== "string" || key.toLowerCase() === "content-length") continue;
res.setHeader(key, value);
}
res.end(typeof response.body === "string" ? response.body : "");
res.end(response.body);
} catch (error) {
writeJsonResponse(res, 502, { error: error instanceof Error ? error.message : String(error) });
}
@@ -2386,40 +2753,47 @@ function runDuplexGateway() {
process.exit(0);
});
// Bind-or-exit. The host assigns a positive loopback port. The gateway binds
// exactly that port. On a non-positive assigned port or a bind failure it
// writes a diagnostic and exits with a nonzero code. It never selects a
// different port, so no untrusted workload can steer the endpoint.
// Bind-or-exit, the same rule every mode applies: the host assigns a
// positive loopback port, and the gateway binds exactly that port or exits
// nonzero. It never selects a different port.
if (!Number.isInteger(port) || port <= 0) {
diag("duplex gateway requires a positive assigned PAPERCLIP_BRIDGE_PORT; got " + String(port));
diag("http2 gateway requires a positive assigned PAPERCLIP_BRIDGE_PORT; got " + String(port));
process.exit(1);
}
server.on("error", (error) => {
diag("duplex gateway could not bind port " + String(port) + ": " + (error && error.message ? error.message : String(error)));
diag(
"http2 gateway could not bind port " +
String(port) +
": " +
(error && error.message ? error.message : String(error)),
);
process.exit(1);
});
server.listen(port, host, () => {
const address = server.address();
if (!address || typeof address === "string") {
diag("duplex gateway did not expose a TCP address");
diag("http2 gateway did not expose a TCP address");
process.exit(1);
return;
}
// The gateway sends READY only after the listener binds. READY is a liveness
// signal: it carries the frame version and the echoed nonce, and no address
// data. The host builds the endpoint from its own stored port. Stdout carries
// only frames.
writeFrame({
version: DUPLEX_FRAME_VERSION,
type: "ready",
nonce: bridgeNonce,
});
// Send the one frame-codec line on this path: the READY line the host
// readiness gate expects. Open the HTTP/2 client session on the very next
// statement, so the client connection preface is the next byte the host
// sees after READY, with no other writer in between.
writeFrame({ version: DUPLEX_FRAME_VERSION, type: "ready", nonce: bridgeNonce });
gatewayReady = true;
openSession();
});
}
if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}") {
runDuplexGateway();
} else {
// The startup check above already rejected every value except http2 and
// queue, so this dispatch names both modes explicitly and never falls
// through to the queue gateway for an unsupported mode.
if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}") {
runHttp2Gateway();
} else if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_FILE_MODE}") {
await runFileGateway();
} else {
throw new Error("Unsupported PAPERCLIP_API_BRIDGE_MODE: " + bridgeMode);
}`;
}
@@ -880,7 +880,12 @@ describe("renderPaperclipWakePrompt", () => {
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("Use child issues");
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("instead of polling agents, sessions, or processes");
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("Create child issues directly when you know what needs to be done");
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("POST /api/issues/{issueId}/interactions");
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("POST /api/issues/$PAPERCLIP_TASK_ID/interactions");
// URL paths in prompt text carry real ids or env vars, never brace
// placeholders: agents paste these lines verbatim, and a literal {issueId}
// reaches the server as /api/issues/%7BissueId%7D and 404s.
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).not.toContain("/api/issues/{issueId}");
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).not.toContain("/api/issues/{id}");
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain("kind suggest_tasks, ask_user_questions, or request_confirmation");
expect(DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE).toContain(
"Use continuationPolicy wake_assignee when you need to resume after a response (it wakes on acceptance and rejection alike; only expiry does not wake); use wake_assignee_on_accept when you want to resume only after acceptance",
+3 -3
View File
@@ -176,9 +176,9 @@ export const DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE = [
"- If woken by a human comment on a dependency-blocked issue, respond or triage the comment without treating the blocked deliverable work as unblocked.",
"- Create child issues directly when you know what needs to be done; use issue-thread interactions when the board/user must choose suggested tasks, answer structured questions, or confirm a proposal.",
"- Use `PAPERCLIP_SCRATCH_DIR` / `PAPERCLIP_RUN_SCRATCH_DIR` for temporary scratch files instead of ad hoc `/tmp` paths; Paperclip removes that run-owned directory after the run ends.",
"- To ask for that input, create an interaction on the current issue with POST /api/issues/{issueId}/interactions using kind suggest_tasks, ask_user_questions, or request_confirmation. Use continuationPolicy wake_assignee when you need to resume after a response (it wakes on acceptance and rejection alike; only expiry does not wake); use wake_assignee_on_accept when you want to resume only after acceptance.",
"- To ask for that input, create an interaction on the current issue with POST /api/issues/$PAPERCLIP_TASK_ID/interactions using kind suggest_tasks, ask_user_questions, or request_confirmation. Use continuationPolicy wake_assignee when you need to resume after a response (it wakes on acceptance and rejection alike; only expiry does not wake); use wake_assignee_on_accept when you want to resume only after acceptance.",
"- Never create probe or throwaway issue-thread interactions to discover the interactions API shape or your permissions; schema discovery goes through the OpenAPI spec and explicit validation errors, not placeholder cards. Every ask_user_questions, suggest_tasks, or request_confirmation you post must carry a real, answerable prompt; withdraw one you no longer need instead of leaving it pending.",
"- When you intentionally restart follow-up work on a completed assigned issue, include structured `resume: true` with the POST /api/issues/{issueId}/comments or PATCH /api/issues/{issueId} comment payload. Generic agent comments on closed issues are inert by default.",
"- When you intentionally restart follow-up work on a completed assigned issue, include structured `resume: true` with the POST /api/issues/$PAPERCLIP_TASK_ID/comments or PATCH /api/issues/$PAPERCLIP_TASK_ID comment payload (substitute that issue's real id when it is not the current task). Generic agent comments on closed issues are inert by default.",
"- For plan approval, update the plan document first, then create request_confirmation targeting the latest plan revision with idempotencyKey confirmation:{issueId}:plan:{revisionId}. Wait for acceptance before creating implementation subtasks, and create a fresh confirmation after superseding board/user comments if approval is still needed.",
"- If blocked, mark the issue blocked and name the unblock owner and action.",
"- Respect budget, pause/cancel, approval gates, and company boundaries.",
@@ -2010,7 +2010,7 @@ export function renderPaperclipWakePrompt(
lines.push(
"",
"The harness already checked out this issue for the current run.",
"Do not call `/api/issues/{id}/checkout` again unless you intentionally switch to a different task.",
"Do not call `POST /api/issues/$PAPERCLIP_TASK_ID/checkout` again unless you intentionally switch to a different task.",
"",
);
}
@@ -106,29 +106,38 @@ function entriesMatch(left: SnapshotEntry | null | undefined, right: SnapshotEnt
return false;
}
async function isHolderAlive(lockDir: string): Promise<boolean> {
const LOCK_STALE_MS = 30_000;
async function isLockStale(lockDir: string): Promise<boolean> {
try {
const raw = await fs.readFile(path.join(lockDir, "owner.json"), "utf8");
const owner = JSON.parse(raw) as { pid?: unknown };
const pid = typeof owner.pid === "number" && Number.isFinite(owner.pid) && owner.pid > 0 ? owner.pid : null;
if (pid === null) {
// Owner record is unparseable / missing pid — treat as stale.
return false;
return true;
}
try {
process.kill(pid, 0);
return true;
} catch {
return false;
} catch {
return true;
}
} catch {
// owner.json missing or unreadable — treat as stale.
return false;
// owner.json is missing or unreadable. A live holder also passes through
// this exact state, briefly, between its own `fs.mkdir(lockDir)` and its
// `fs.writeFile(owner.json)` below. Reading "missing" as "stale" here would
// let a concurrent acquirer delete a live holder's lock directory during
// that window. Mirror the materializePaperclipSkillCopy lock pattern: fall
// back to the lock directory's own mtime, and only call it stale once the
// directory itself has outlived the stale threshold.
const stat = await fs.stat(lockDir).catch(() => null);
return !stat || Date.now() - stat.mtimeMs > LOCK_STALE_MS;
}
}
async function acquireDirectoryMergeLock(lockDir: string): Promise<() => Promise<void>> {
const deadline = Date.now() + 30_000;
const deadline = Date.now() + LOCK_STALE_MS;
while (true) {
try {
await fs.mkdir(lockDir);
@@ -146,7 +155,7 @@ async function acquireDirectoryMergeLock(lockDir: string): Promise<() => Promise
// Stale-lock detection: if the owner PID is dead (SIGKILL / OOM / crash),
// the lockDir would otherwise persist forever and stall restores. Mirror
// the materializePaperclipSkillCopy lock pattern — remove and retry.
if (!(await isHolderAlive(lockDir))) {
if (await isLockStale(lockDir)) {
await fs.rm(lockDir, { recursive: true, force: true }).catch(() => undefined);
continue;
}
+1 -1
View File
@@ -53,7 +53,7 @@
"probe:quota:raw": "pnpm exec tsx src/cli/quota-probe.ts --json --raw-cli"
},
"dependencies": {
"@agentclientprotocol/claude-agent-acp": "^0.69.0",
"@agentclientprotocol/claude-agent-acp": "^0.70.0",
"@paperclipai/adapter-utils": "workspace:*",
"@paperclipai/shared": "workspace:*",
"picocolors": "^1.1.1"
@@ -15,7 +15,7 @@ import {
ensureAdapterExecutionTargetCommandResolvable,
ensureAdapterExecutionTargetRuntimeCommandInstalled,
prepareAdapterExecutionTargetRuntime,
adapterExecutionTargetDuplexTelemetryRecorder,
adapterExecutionTargetDuplexObservabilityRecorder,
adapterExecutionTargetEnablesSandboxDuplexBridge,
readAdapterExecutionTarget,
resolveAdapterExecutionTargetTimeoutSec,
@@ -684,7 +684,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
runId,
target: runtimeExecutionTarget,
enableSandboxDuplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(runtimeExecutionTarget),
duplexTelemetryRecorder: adapterExecutionTargetDuplexTelemetryRecorder(runtimeExecutionTarget),
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(runtimeExecutionTarget),
runtimeRootDir: preparedExecutionTargetRuntime?.runtimeRootDir,
adapterKey: "claude",
timeoutSec,
+1 -1
View File
@@ -52,7 +52,7 @@
"probe:quota": "pnpm exec tsx src/cli/quota-probe.ts --json"
},
"dependencies": {
"@agentclientprotocol/codex-acp": "^1.2.0",
"@agentclientprotocol/codex-acp": "^1.6.2",
"@paperclipai/adapter-utils": "workspace:*",
"@paperclipai/shared": "workspace:*",
"picocolors": "^1.1.1"
@@ -42,6 +42,62 @@ describe("parseDeviceLoginPrompt", () => {
expect(result?.code).toBe("ABCD-EFGHJ");
});
it("parse_returns_url_and_code_from_carriage_return_line_endings", () => {
// A pseudo-terminal ends each line with a carriage return and a line feed.
// The carriage return is whitespace, so the URL token stops before it, and the
// code line trims it. The parser reads the tokens the same as line-feed output.
const text = [
"1. Open this link in your browser and sign in to your account",
EXACT_URL,
"2. Enter this one-time code (expires in 15 minutes)",
"ABCD-EFGHJ",
].join("\r\n");
const result = parseDeviceLoginPrompt(text);
expect(result).not.toBeNull();
expect(result?.url).toBe(EXACT_URL);
expect(result?.code).toBe("ABCD-EFGHJ");
});
it("parse_returns_url_and_code_with_a_terminal_title_osc_sequence", () => {
// A pseudo-terminal can emit an Operating System Command (OSC) title sequence
// in the stream. The device-auth prompt still prints the URL and the code as
// plain standalone tokens on their own lines. The OSC title sequence carries no
// URL token, so it does not shift the URL or the code, and the parser reads
// both tokens.
const osc = "\x1b]0;Codex\x07";
const text = [
`${osc}1. Open this link in your browser and sign in to your account`,
EXACT_URL,
"2. Enter this one-time code (expires in 15 minutes)",
"ABCD-EFGHJ",
].join("\n");
const result = parseDeviceLoginPrompt(text);
expect(result).not.toBeNull();
expect(result?.url).toBe(EXACT_URL);
expect(result?.code).toBe("ABCD-EFGHJ");
});
it("parse_returns_url_and_code_from_a_merged_raw_pseudo_terminal_frame", () => {
// One buffer holds the whole prompt as a real pseudo-terminal emits it: a
// leading title Operating System Command (OSC) sequence, color Control
// Sequence Introducer (CSI) sequences around the URL and the code, and
// carriage-return line endings. The parser removes the color sequences and
// reads the plain tokens.
const title = "\x1b]0;Codex login\x07";
const cyan = "\x1b[36m";
const bold = "\x1b[1m";
const reset = "\x1b[0m";
const text =
`${title}1. Open this link in your browser and sign in to your account\r\n` +
`${cyan}${EXACT_URL}${reset}\r\n` +
`2. Enter this one-time code (expires in 15 minutes)\r\n` +
`${bold}ABCD-EFGHJ${reset}\r\n`;
const result = parseDeviceLoginPrompt(text);
expect(result).not.toBeNull();
expect(result?.url).toBe(EXACT_URL);
expect(result?.code).toBe("ABCD-EFGHJ");
});
it("parse_returns_null_when_prompt_absent", () => {
const text = "Some unrelated log line\nNothing to see here\n";
expect(parseDeviceLoginPrompt(text)).toBeNull();
@@ -24,10 +24,10 @@ interface FakeDriverOptions {
function createFakeDriver(options: FakeDriverOptions = {}) {
const disposeCalls = { count: 0 };
const driver: SandboxLoginDriver = {
async execStreaming(_command, onStdout) {
async start(_command, onData) {
if (options.execError) throw options.execError;
for (const chunk of options.chunks ?? []) {
onStdout(chunk);
onData(chunk);
}
if (options.hang) {
// Never resolve. The runner must fall back to its timeout or signal.
@@ -38,18 +38,33 @@ const MAX_PARSE_BUFFER_CHARS = 64 * 1024;
/**
* The sandbox side of the device-login run. The runner never calls Daytona
* directly; a caller injects a concrete driver. A production driver binds these
* three methods to a non-persisting Daytona exec path, a file read, and a
* sandbox delete.
* directly; a caller injects a concrete driver. A production driver binds `start`
* to the shared login pseudo-terminal (PTY) transport, binds `readFile` to a
* descriptor-bound credential read, and binds `dispose` to the transport dispose.
*
* The `start` shape matches the shared login pseudo-terminal transport start
* method, so the driver runs the login command on a real pseudo-terminal. The
* login command
* needs a pseudo-terminal: pipe stdio emits no login prompt. The runner never
* calls `stop` or `write` on the driver; the device-login flow needs no delayed
* input, and the service owns the sandbox delete. The runner disposes the driver
* one time on every terminal state.
*/
export interface SandboxLoginDriver extends LoginRunnerDisposable {
/**
* Runs `command` in the sandbox and streams standard output to `onStdout` in
* memory. Resolves with the command exit code when the command ends. A driver
* must not persist the raw output to any durable log.
* Starts `command` on a pseudo-terminal and streams the terminal output to
* `onData` in memory, in order, as the pseudo-terminal emits it. Resolves with
* the command exit code when the command ends. A driver must not persist the raw
* output to any durable log.
*/
start(command: string, onData: (chunk: string) => void): Promise<{ exitCode: number | null }>;
/**
* Reads the credential bytes with one descriptor-bound read. The read is
* separate from the pseudo-terminal session; the session has no file-read
* method. A driver runs one fixed, server-controlled operation that opens the
* verified session home and the credential file with no symlink follow, checks
* the opened descriptor, and reads only from that same descriptor.
*/
execStreaming(command: string, onStdout: (chunk: string) => void): Promise<{ exitCode: number | null }>;
/** Reads the bytes of one file from the sandbox. */
readFile(path: string): Promise<Buffer>;
}
@@ -123,8 +138,8 @@ export async function runDeviceLogin(
return { outcome: "cancelled", exitCode: null, promptSurfaced };
}
const exec = driver.execStreaming(command, onStdout);
const raced = await raceLoginRunnerExit(exec, timeoutMs, signal);
const started = driver.start(command, onStdout);
const raced = await raceLoginRunnerExit(started, timeoutMs, signal);
if (raced.kind === "timeout") {
log("[paperclip] Device login timed out; disposing the sandbox.");
@@ -21,7 +21,7 @@ import {
ensureAdapterExecutionTargetCommandResolvable,
ensureAdapterExecutionTargetRuntimeCommandInstalled,
prepareAdapterExecutionTargetRuntime,
adapterExecutionTargetDuplexTelemetryRecorder,
adapterExecutionTargetDuplexObservabilityRecorder,
adapterExecutionTargetEnablesSandboxDuplexBridge,
readAdapterExecutionTarget,
resolveAdapterExecutionTargetTimeoutSec,
@@ -954,7 +954,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
runId,
target: runtimeExecutionTarget,
enableSandboxDuplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(runtimeExecutionTarget),
duplexTelemetryRecorder: adapterExecutionTargetDuplexTelemetryRecorder(runtimeExecutionTarget),
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(runtimeExecutionTarget),
runtimeRootDir: preparedExecutionTargetRuntime?.runtimeRootDir,
adapterKey: "codex",
timeoutSec,
@@ -15,7 +15,7 @@ import {
ensureAdapterExecutionTargetCommandResolvable,
ensureAdapterExecutionTargetRuntimeCommandInstalled,
prepareAdapterExecutionTargetRuntime,
adapterExecutionTargetDuplexTelemetryRecorder,
adapterExecutionTargetDuplexObservabilityRecorder,
adapterExecutionTargetEnablesSandboxDuplexBridge,
readAdapterExecutionTarget,
readAdapterExecutionTargetHomeDir,
@@ -459,7 +459,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
runId,
target: runtimeExecutionTarget,
enableSandboxDuplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(runtimeExecutionTarget),
duplexTelemetryRecorder: adapterExecutionTargetDuplexTelemetryRecorder(runtimeExecutionTarget),
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(runtimeExecutionTarget),
runtimeRootDir: remoteRuntimeRootDir,
adapterKey: "cursor",
timeoutSec,
@@ -16,7 +16,7 @@ import {
ensureAdapterExecutionTargetCommandResolvable,
ensureAdapterExecutionTargetRuntimeCommandInstalled,
prepareAdapterExecutionTargetRuntime,
adapterExecutionTargetDuplexTelemetryRecorder,
adapterExecutionTargetDuplexObservabilityRecorder,
adapterExecutionTargetEnablesSandboxDuplexBridge,
readAdapterExecutionTarget,
readAdapterExecutionTargetHomeDir,
@@ -124,7 +124,8 @@ function renderApiAccessNote(env: Record<string, string>): string {
"GET example:",
` run_shell_command({ command: "curl -s -H \\"Authorization: Bearer $PAPERCLIP_API_KEY\\" \\"$PAPERCLIP_API_URL/api/agents/me\\"" })`,
"POST/PATCH example:",
` run_shell_command({ command: "curl -s -X POST -H \\"Authorization: Bearer $PAPERCLIP_API_KEY\\" -H 'Content-Type: application/json' -H \\"X-Paperclip-Run-Id: $PAPERCLIP_RUN_ID\\" -d '{...}' \\"$PAPERCLIP_API_URL/api/issues/{id}/checkout\\"" })`,
` run_shell_command({ command: "curl -s -X POST -H \\"Authorization: Bearer $PAPERCLIP_API_KEY\\" -H 'Content-Type: application/json' -H \\"X-Paperclip-Run-Id: $PAPERCLIP_RUN_ID\\" -d '{...}' \\"$PAPERCLIP_API_URL/api/issues/$PAPERCLIP_TASK_ID/checkout\\"" })`,
"When PAPERCLIP_TASK_ID is not set, substitute a real issue id from the current context; never send a placeholder like {id} in the URL.",
"",
"",
].join("\n");
@@ -461,7 +462,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
runId,
target: runtimeExecutionTarget,
enableSandboxDuplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(runtimeExecutionTarget),
duplexTelemetryRecorder: adapterExecutionTargetDuplexTelemetryRecorder(runtimeExecutionTarget),
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(runtimeExecutionTarget),
runtimeRootDir: remoteRuntimeRootDir,
adapterKey: "gemini",
timeoutSec,
+1 -1
View File
@@ -1,5 +1,5 @@
{
"extends": "../../../tsconfig.json",
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src"
@@ -105,6 +105,84 @@ describe("grok_local testEnvironment", () => {
);
});
it("does not warn when the default-sentinel model is absent from the real model list", async () => {
// Real grok never lists the "grok-build" sentinel; execute.ts never sends
// it. The probe must treat the default as valid (info), not warn.
runProcessMock
.mockResolvedValueOnce({
exitCode: 0,
signal: null,
timedOut: false,
stdout: [
"You are logged in with grok.com.",
"",
"Default model: grok-4.20-0309-non-reasoning",
"",
"Available models:",
" * grok-4.20-0309-non-reasoning (default)",
" * grok-4",
].join("\n"),
stderr: "",
})
.mockResolvedValueOnce({
exitCode: 0,
signal: null,
timedOut: false,
stdout: [
JSON.stringify({ type: "text", data: "hello" }),
JSON.stringify({ type: "end", stopReason: "EndTurn", sessionId: "s", requestId: "r" }),
].join("\n"),
stderr: "",
});
const result = await testEnvironment({
companyId: "company-1",
adapterType: "grok_local",
config: { command: "grok", cwd: "/tmp/project" }, // no model → default sentinel
});
const codes = result.checks.map((check: { code: string }) => check.code);
expect(codes).toContain("grok_model_configured");
expect(codes).not.toContain("grok_model_not_found");
expect(result.status).toBe("pass");
});
it("warns when an explicitly configured real model is not available", async () => {
runProcessMock
.mockResolvedValueOnce({
exitCode: 0,
signal: null,
timedOut: false,
stdout: [
"You are logged in with grok.com.",
"",
"Default model: grok-4",
"",
"Available models:",
" * grok-4",
].join("\n"),
stderr: "",
})
.mockResolvedValueOnce({
exitCode: 0,
signal: null,
timedOut: false,
stdout: [
JSON.stringify({ type: "text", data: "hello" }),
JSON.stringify({ type: "end", stopReason: "EndTurn", sessionId: "s", requestId: "r" }),
].join("\n"),
stderr: "",
});
const result = await testEnvironment({
companyId: "company-1",
adapterType: "grok_local",
config: { command: "grok", cwd: "/tmp/project", model: "grok-nonexistent" },
});
expect(result.checks.map((check: { code: string }) => check.code)).toContain("grok_model_not_found");
});
it("downgrades auth failures to warnings", async () => {
runProcessMock
.mockResolvedValueOnce({
@@ -226,15 +226,22 @@ export async function testEnvironment(
});
}
if (configuredModel) {
// The default model id is a sentinel meaning "let the Grok CLI pick its
// own default" — execute.ts only passes `--model` when the value differs
// from it, so the sentinel is never sent to grok and must not be checked
// against the discovered list (real grok never lists "grok-build", which
// otherwise produced a spurious "not found" warning on every probe).
const usesCliDefault = configuredModel === DEFAULT_GROK_LOCAL_MODEL;
const available = usesCliDefault || parsedModels.models.includes(configuredModel);
checks.push({
code: parsedModels.models.includes(configuredModel) ? "grok_model_configured" : "grok_model_not_found",
level: parsedModels.models.includes(configuredModel) ? "info" : "warn",
message: parsedModels.models.includes(configuredModel)
? `Configured model: ${configuredModel}`
: `Configured model "${configuredModel}" not found in available models.`,
hint: parsedModels.models.includes(configuredModel)
? undefined
: "Run `grok models` and choose an available model id.",
code: available ? "grok_model_configured" : "grok_model_not_found",
level: available ? "info" : "warn",
message: usesCliDefault
? `Using the Grok CLI's default model${parsedModels.defaultModel ? ` (${parsedModels.defaultModel})` : ""}.`
: available
? `Configured model: ${configuredModel}`
: `Configured model "${configuredModel}" not found in available models.`,
hint: available ? undefined : "Run `grok models` and choose an available model id.",
});
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
{
"extends": "../../../tsconfig.json",
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src"
@@ -423,6 +423,7 @@ function buildWakeText(
"Workflow:",
"1) GET /api/agents/me",
`2) Determine issueId: PAPERCLIP_TASK_ID if present, otherwise issue_id (${issueIdHint}).`,
' Replace {issueId} in every endpoint below with that determined id. Never send the literal text "{issueId}" in a URL.',
"3) If issueId exists:",
" - POST /api/issues/{issueId}/checkout with {\"agentId\":\"$PAPERCLIP_AGENT_ID\",\"expectedStatuses\":[\"todo\",\"backlog\",\"blocked\",\"in_review\"]}",
" - GET /api/issues/{issueId}",
@@ -15,7 +15,7 @@ import {
ensureAdapterExecutionTargetCommandResolvable,
ensureAdapterExecutionTargetRuntimeCommandInstalled,
prepareAdapterExecutionTargetRuntime,
adapterExecutionTargetDuplexTelemetryRecorder,
adapterExecutionTargetDuplexObservabilityRecorder,
adapterExecutionTargetEnablesSandboxDuplexBridge,
readAdapterExecutionTarget,
readAdapterExecutionTargetHomeDir,
@@ -463,7 +463,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
runId,
target: runtimeExecutionTarget,
enableSandboxDuplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(runtimeExecutionTarget),
duplexTelemetryRecorder: adapterExecutionTargetDuplexTelemetryRecorder(runtimeExecutionTarget),
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(runtimeExecutionTarget),
runtimeRootDir: remoteRuntimeRootDir,
adapterKey: "opencode",
timeoutSec,
@@ -16,7 +16,7 @@ import {
ensureAdapterExecutionTargetFile,
ensureAdapterExecutionTargetRuntimeCommandInstalled,
prepareAdapterExecutionTargetRuntime,
adapterExecutionTargetDuplexTelemetryRecorder,
adapterExecutionTargetDuplexObservabilityRecorder,
adapterExecutionTargetEnablesSandboxDuplexBridge,
readAdapterExecutionTarget,
resolveAdapterExecutionTargetTimeoutSec,
@@ -472,7 +472,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
runId,
target: runtimeExecutionTarget,
enableSandboxDuplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(runtimeExecutionTarget),
duplexTelemetryRecorder: adapterExecutionTargetDuplexTelemetryRecorder(runtimeExecutionTarget),
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(runtimeExecutionTarget),
runtimeRootDir: remoteRuntimeRootDir,
adapterKey: "pi",
timeoutSec,
+27
View File
@@ -140,6 +140,21 @@ describeEmbeddedPostgres("runDatabaseBackup", () => {
"created_at" timestamptz NOT NULL DEFAULT now()
);
`);
await sourceSql.unsafe(`
CREATE FUNCTION "public"."backup_test_mark_done"()
RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
NEW."state" := 'done';
RETURN NEW;
END;
$$;
CREATE TRIGGER "backup_test_mark_done_trigger"
BEFORE UPDATE OF "title" ON "public"."backup_test_records"
FOR EACH ROW
EXECUTE FUNCTION "public"."backup_test_mark_done"();
`);
const payload = "x".repeat(8192);
for (let index = 0; index < 160; index += 1) {
@@ -213,6 +228,18 @@ describeEmbeddedPostgres("runDatabaseBackup", () => {
metadata: { index: 159, even: false },
},
]);
await restoreSql.unsafe(`
UPDATE "public"."backup_test_records"
SET "title" = 'triggered'
WHERE "title" = 'row-0'
`);
const triggeredRows = await restoreSql.unsafe<{ state: string }[]>(`
SELECT "state"::text AS "state"
FROM "public"."backup_test_records"
WHERE "title" = 'triggered'
`);
expect(triggeredRows).toEqual([{ state: "done" }]);
} finally {
await sourceSql.end();
await restoreSql.end();
+66 -9
View File
@@ -591,6 +591,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
emitStatement("BEGIN;");
emitStatement("SET LOCAL session_replication_role = replica;");
emitStatement("SET LOCAL client_min_messages = warning;");
emitStatement("SET LOCAL check_function_bodies = false;");
emit("");
const allTables = await sql<TableDefinition[]>`
@@ -809,7 +810,9 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
emit("");
}
// Foreign keys (after all tables and referenced unique constraints are created)
// Collect foreign keys now. Emit them after routines and standalone indexes
// because PostgreSQL permits a non-constraint unique index to be the target
// of a foreign key.
const allForeignKeys = await sql<{
constraint_name: string;
source_schema: string;
@@ -849,14 +852,29 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
&& includedTableNames.has(tableKey(fk.target_schema, fk.target_table)),
);
if (fks.length > 0) {
emit("-- Foreign keys");
for (const fk of fks) {
const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", ");
const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", ");
emitStatement(
`ALTER TABLE ${quoteQualifiedName(fk.source_schema, fk.source_table)} ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES ${quoteQualifiedName(fk.target_schema, fk.target_table)} (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`,
);
// JavaScript backups are used when a worktree seed filters or transforms
// table data. Preserve user-defined routines before indexes because an
// expression index may depend on a user-defined function.
const routines = await sql<{ definition: string }[]>`
SELECT pg_get_functiondef(p.oid) AS definition
FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
WHERE ${sql.unsafe(nonSystemSchemaPredicate("n.nspname"))}
AND p.prokind IN ('f', 'p')
AND NOT EXISTS (
SELECT 1
FROM pg_depend d
WHERE d.classid = 'pg_proc'::regclass
AND d.objid = p.oid
AND d.deptype = 'e'
)
ORDER BY n.nspname, p.proname, pg_get_function_identity_arguments(p.oid)
`;
if (routines.length > 0) {
emit("-- Functions and procedures");
for (const routine of routines) {
const definition = routine.definition.trimEnd();
emitStatement(definition.endsWith(";") ? definition : `${definition};`);
}
emit("");
}
@@ -883,6 +901,18 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
emit("");
}
if (fks.length > 0) {
emit("-- Foreign keys");
for (const fk of fks) {
const srcCols = fk.source_columns.map((c) => `"${c}"`).join(", ");
const tgtCols = fk.target_columns.map((c) => `"${c}"`).join(", ");
emitStatement(
`ALTER TABLE ${quoteQualifiedName(fk.source_schema, fk.source_table)} ADD CONSTRAINT "${fk.constraint_name}" FOREIGN KEY (${srcCols}) REFERENCES ${quoteQualifiedName(fk.target_schema, fk.target_table)} (${tgtCols}) ON UPDATE ${fk.update_rule} ON DELETE ${fk.delete_rule};`,
);
}
emit("");
}
// Dump data for each table
for (const { schema_name, tablename } of tables) {
const currentTableKey = tableKey(schema_name, tablename);
@@ -938,6 +968,33 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise
emit("");
}
const allTriggers = await sql<{
schema_name: string;
tablename: string;
definition: string;
}[]>`
SELECT
n.nspname AS schema_name,
c.relname AS tablename,
pg_get_triggerdef(t.oid, true) AS definition
FROM pg_trigger t
JOIN pg_class c ON c.oid = t.tgrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE NOT t.tgisinternal
AND ${sql.unsafe(nonSystemSchemaPredicate("n.nspname"))}
ORDER BY n.nspname, c.relname, t.tgname
`;
const triggers = allTriggers.filter((entry) => (
includedTableNames.has(tableKey(entry.schema_name, entry.tablename))
));
if (triggers.length > 0) {
emit("-- Triggers");
for (const trigger of triggers) {
emitStatement(`${trigger.definition};`);
}
emit("");
}
// Sequence values
if (sequences.length > 0) {
emit("-- Sequence values");
+462
View File
@@ -1405,4 +1405,466 @@ describeEmbeddedPostgres("applyPendingMigrations", () => {
},
20_000,
);
it(
"preserves legacy runs while adding native persistence and replay-safe status versioning",
async () => {
const connectionString = await createTempDatabase();
await applyPendingMigrations(connectionString);
const nativePersistenceHash = await migrationHash("0227_modern_pandemic.sql");
const sql = postgres(connectionString, { max: 1, onnotice: () => {} });
const companyId = "10000000-0000-4000-8000-000000000227";
const agentId = "20000000-0000-4000-8000-000000000227";
const runId = "30000000-0000-4000-8000-000000000227";
const issueId = "40000000-0000-4000-8000-000000000227";
const contractId = "50000000-0000-4000-8000-000000000227";
const resultId = "60000000-0000-4000-8000-000000000227";
const assessmentId = "70000000-0000-4000-8000-000000000227";
const decisionId = "80000000-0000-4000-8000-000000000227";
const otherCompanyId = "11000000-0000-4000-8000-000000000227";
const otherAgentId = "21000000-0000-4000-8000-000000000227";
const otherRunId = "31000000-0000-4000-8000-000000000227";
const otherIssueId = "41000000-0000-4000-8000-000000000227";
const otherContractId = "51000000-0000-4000-8000-000000000227";
const otherResultId = "61000000-0000-4000-8000-000000000227";
const otherAssessmentId = "71000000-0000-4000-8000-000000000227";
const otherDecisionId = "81000000-0000-4000-8000-000000000227";
try {
await sql.unsafe(`
DROP TABLE IF EXISTS status_decision_effects, status_decisions, work_assessments,
native_run_finalizations, native_run_results, completion_contracts CASCADE;
DROP TRIGGER IF EXISTS paperclip_issue_status_version_trigger ON issues;
DROP FUNCTION IF EXISTS paperclip_bump_issue_status_version();
DROP INDEX IF EXISTS issues_company_id_uq;
DROP INDEX IF EXISTS heartbeat_run_events_run_source_event_uq;
DROP INDEX IF EXISTS heartbeat_run_events_run_source_seq_uq;
ALTER TABLE heartbeat_run_events
DROP COLUMN IF EXISTS source_instance_id,
DROP COLUMN IF EXISTS source_event_id,
DROP COLUMN IF EXISTS source_seq,
DROP COLUMN IF EXISTS source_payload_sha256,
DROP COLUMN IF EXISTS protocol_schema_version;
ALTER TABLE heartbeat_run_events ALTER COLUMN seq TYPE integer;
ALTER TABLE heartbeat_runs
DROP COLUMN IF EXISTS runtime_mode,
DROP COLUMN IF EXISTS runtime_mode_resolver_version,
DROP COLUMN IF EXISTS runtime_mode_reason,
DROP COLUMN IF EXISTS runtime_mode_resolved_at,
DROP COLUMN IF EXISTS runner_profile_json,
DROP COLUMN IF EXISTS runner_instance_id,
DROP COLUMN IF EXISTS native_session_id,
DROP COLUMN IF EXISTS native_issue_id,
DROP COLUMN IF EXISTS driver_kind,
DROP COLUMN IF EXISTS driver_version,
DROP COLUMN IF EXISTS completion_contract_id,
DROP COLUMN IF EXISTS completion_contract_sha256,
DROP COLUMN IF EXISTS next_event_seq,
DROP COLUMN IF EXISTS native_phase,
DROP COLUMN IF EXISTS native_phase_updated_at;
ALTER TABLE issues
DROP COLUMN IF EXISTS status_version,
DROP COLUMN IF EXISTS last_status_decision_id;
`);
await sql`DELETE FROM "drizzle"."__drizzle_migrations" WHERE "hash" = ${nativePersistenceHash}`;
await sql`
INSERT INTO companies (id, name, issue_prefix)
VALUES (${companyId}, 'Native persistence fixture', 'NPF')
`;
await sql`
INSERT INTO agents (id, company_id, name)
VALUES (${agentId}, ${companyId}, 'Legacy migration agent')
`;
await sql`
INSERT INTO heartbeat_runs (id, company_id, agent_id, status)
VALUES (${runId}, ${companyId}, ${agentId}, 'succeeded')
`;
await sql`
INSERT INTO issues (id, company_id, title, status)
VALUES (${issueId}, ${companyId}, 'Legacy migration issue', 'in_progress')
`;
await sql.unsafe(`
INSERT INTO heartbeat_run_events
(company_id, run_id, agent_id, seq, event_type, stream, level, message, payload, created_at)
VALUES
('${companyId}', '${runId}', '${agentId}', 1, 'legacy.start', 'system', 'info', 'one', '{"bytes":"alpha-1"}'::jsonb, '2026-08-01T00:00:01.000Z'),
('${companyId}', '${runId}', '${agentId}', 5, 'legacy.log', 'stdout', 'info', 'first-five', '{"bytes":"beta-5a"}'::jsonb, '2026-08-01T00:00:02.000Z'),
('${companyId}', '${runId}', '${agentId}', 5, 'legacy.log', 'stderr', 'warn', 'duplicate-five', '{"bytes":"gamma-5b"}'::jsonb, '2026-08-01T00:00:03.000Z'),
('${companyId}', '${runId}', '${agentId}', 9, 'legacy.end', 'system', 'info', 'nine', '{"bytes":"delta-9"}'::jsonb, '2026-08-01T00:00:04.000Z')
`);
} finally {
await sql.end();
}
await applyPendingMigrations(connectionString);
const verifySql = postgres(connectionString, { max: 1, onnotice: () => {} });
try {
const events = await verifySql.unsafe<{
seq: string;
event_type: string;
stream: string;
level: string;
message: string;
payload: { bytes: string };
created_at: Date;
}[]>(`
SELECT seq, event_type, stream, level, message, payload, created_at
FROM heartbeat_run_events
WHERE run_id = '${runId}'
ORDER BY id
`);
expect(events.map((event) => Number(event.seq))).toEqual([1, 5, 5, 9]);
expect(events.map(({ seq: _seq, ...event }) => ({
...event,
created_at: event.created_at.toISOString(),
}))).toEqual([
{
event_type: "legacy.start",
stream: "system",
level: "info",
message: "one",
payload: { bytes: "alpha-1" },
created_at: "2026-08-01T00:00:01.000Z",
},
{
event_type: "legacy.log",
stream: "stdout",
level: "info",
message: "first-five",
payload: { bytes: "beta-5a" },
created_at: "2026-08-01T00:00:02.000Z",
},
{
event_type: "legacy.log",
stream: "stderr",
level: "warn",
message: "duplicate-five",
payload: { bytes: "gamma-5b" },
created_at: "2026-08-01T00:00:03.000Z",
},
{
event_type: "legacy.end",
stream: "system",
level: "info",
message: "nine",
payload: { bytes: "delta-9" },
created_at: "2026-08-01T00:00:04.000Z",
},
]);
const runs = await verifySql.unsafe<{ runtime_mode: string; next_event_seq: string }[]>(`
SELECT runtime_mode, next_event_seq
FROM heartbeat_runs
WHERE id = '${runId}'
`);
expect(runs.map((run) => ({
runtimeMode: run.runtime_mode,
nextEventSeq: Number(run.next_event_seq),
}))).toEqual([{ runtimeMode: "legacy", nextEventSeq: 10 }]);
const nativeRowsBefore = await verifySql.unsafe<{ table_name: string; row_count: number }[]>(`
SELECT 'completion_contracts' AS table_name, count(*)::int AS row_count FROM completion_contracts
UNION ALL SELECT 'native_run_results', count(*)::int FROM native_run_results
UNION ALL SELECT 'native_run_finalizations', count(*)::int FROM native_run_finalizations
UNION ALL SELECT 'work_assessments', count(*)::int FROM work_assessments
UNION ALL SELECT 'status_decisions', count(*)::int FROM status_decisions
UNION ALL SELECT 'status_decision_effects', count(*)::int FROM status_decision_effects
ORDER BY table_name
`);
expect(nativeRowsBefore.every((row) => row.row_count === 0)).toBe(true);
await verifySql`
INSERT INTO companies (id, name, issue_prefix)
VALUES (${otherCompanyId}, 'Other native persistence fixture', 'ONP')
`;
await verifySql`
INSERT INTO agents (id, company_id, name)
VALUES (${otherAgentId}, ${otherCompanyId}, 'Other native migration agent')
`;
await verifySql`
INSERT INTO heartbeat_runs (
id, company_id, agent_id, status, native_issue_id, completion_contract_id
) VALUES (
${otherRunId}, ${otherCompanyId}, ${otherAgentId}, 'succeeded',
${otherIssueId}, ${otherContractId}
)
`;
await verifySql`
INSERT INTO issues (id, company_id, title, status)
VALUES (${otherIssueId}, ${otherCompanyId}, 'Other native issue', 'in_progress')
`;
await verifySql`
UPDATE heartbeat_runs
SET native_issue_id = ${issueId}, completion_contract_id = ${contractId}
WHERE id = ${runId}
`;
await expect(verifySql`
INSERT INTO completion_contracts (
company_id, issue_id, revision, schema_version, policy_version,
risk, completion_authority, incomplete_criteria_policy, contract_json,
canonical_sha256, created_by_actor_type, created_by_actor_id
) VALUES (
${companyId}, ${otherIssueId}, 1, 'paperclip.completion-contract.v1',
'policy-v1', 'low', 'server', 'review', ${JSON.stringify({ criteria: [] })}::jsonb,
'cross-company-contract-sha', 'system', 'migration-test'
)
`).rejects.toThrow(/completion_contracts_issue_company_fk/);
await verifySql`
INSERT INTO completion_contracts (
id, company_id, issue_id, revision, schema_version, policy_version,
risk, completion_authority, incomplete_criteria_policy, contract_json,
canonical_sha256, created_by_actor_type, created_by_actor_id
) VALUES (
${contractId}, ${companyId}, ${issueId}, 1, 'paperclip.completion-contract.v1',
'policy-v1', 'low', 'server', 'review', ${JSON.stringify({ criteria: [] })}::jsonb,
'contract-sha', 'system', 'migration-test'
)
`;
await verifySql`
INSERT INTO completion_contracts (
id, company_id, issue_id, revision, schema_version, policy_version,
risk, completion_authority, incomplete_criteria_policy, contract_json,
canonical_sha256, created_by_actor_type, created_by_actor_id
) VALUES (
${otherContractId}, ${otherCompanyId}, ${otherIssueId}, 1,
'paperclip.completion-contract.v1', 'policy-v1', 'low', 'server', 'review',
${JSON.stringify({ criteria: [] })}::jsonb, 'other-contract-sha',
'system', 'migration-test'
)
`;
await expect(verifySql`
INSERT INTO native_run_results (
company_id, issue_id, run_id, completion_contract_id,
server_fingerprint, schema_status, result_json, canonical_sha256
) VALUES (
${companyId}, ${issueId}, ${otherRunId}, ${contractId},
'cross-company-run', 'valid', ${JSON.stringify({ summary: "invalid" })}::jsonb,
'cross-company-run-sha'
)
`).rejects.toThrow(/native_run_results_run_contract_owner_fk/);
await expect(verifySql`
INSERT INTO native_run_results (
company_id, issue_id, run_id, completion_contract_id,
server_fingerprint, schema_status, result_json, canonical_sha256
) VALUES (
${companyId}, ${issueId}, ${runId}, ${otherContractId},
'cross-company-contract', 'valid', ${JSON.stringify({ summary: "invalid" })}::jsonb,
'cross-company-contract-sha'
)
`).rejects.toThrow(/native_run_results_run_contract_owner_fk/);
await verifySql`
INSERT INTO native_run_results (
id, company_id, issue_id, run_id, completion_contract_id,
server_fingerprint, schema_status, result_json, canonical_sha256
) VALUES (
${resultId}, ${companyId}, ${issueId}, ${runId}, ${contractId},
'result-fingerprint', 'valid', ${JSON.stringify({ summary: "done" })}::jsonb, 'result-sha'
)
`;
await verifySql`
INSERT INTO native_run_results (
id, company_id, issue_id, run_id, completion_contract_id,
server_fingerprint, schema_status, result_json, canonical_sha256
) VALUES (
${otherResultId}, ${otherCompanyId}, ${otherIssueId}, ${otherRunId},
${otherContractId}, 'other-result-fingerprint', 'valid',
${JSON.stringify({ summary: "other" })}::jsonb, 'other-result-sha'
)
`;
await expect(verifySql`
INSERT INTO work_assessments (
company_id, issue_id, run_id, contract_id, result_id,
trigger_kind, trigger_actor_company_id, prior_issue_status,
prior_status_version, policy_version, assessment_json, input_digest
) VALUES (
${companyId}, ${issueId}, ${runId}, ${contractId}, ${otherResultId},
'run_terminal', ${companyId}, 'in_progress', 0, 'policy-v1',
${JSON.stringify({ disposition: "invalid" })}::jsonb,
'cross-company-assessment-input-sha'
)
`).rejects.toThrow(/work_assessments_result_owner_fk/);
await expect(verifySql`
INSERT INTO work_assessments (
company_id, issue_id, run_id, contract_id, result_id,
trigger_kind, trigger_actor_company_id, prior_issue_status,
prior_status_version, policy_version, assessment_json, input_digest
) VALUES (
${companyId}, ${issueId}, ${runId}, ${contractId}, ${resultId},
'run_terminal', ${otherCompanyId}, 'in_progress', 0, 'policy-v1',
${JSON.stringify({ disposition: "invalid" })}::jsonb,
'cross-company-trigger-input-sha'
)
`).rejects.toThrow(/work_assessments_trigger_actor_company_check/);
await verifySql`
INSERT INTO work_assessments (
id, company_id, issue_id, run_id, contract_id, result_id,
trigger_kind, trigger_actor_company_id, prior_issue_status,
prior_status_version, policy_version, assessment_json, input_digest
) VALUES (
${assessmentId}, ${companyId}, ${issueId}, ${runId}, ${contractId}, ${resultId},
'run_terminal', ${companyId}, 'in_progress', 0, 'policy-v1',
${JSON.stringify({ disposition: "done" })}::jsonb, 'assessment-input-sha'
)
`;
await verifySql`
INSERT INTO work_assessments (
id, company_id, issue_id, run_id, contract_id, result_id,
trigger_kind, trigger_actor_company_id, prior_issue_status,
prior_status_version, policy_version, assessment_json, input_digest
) VALUES (
${otherAssessmentId}, ${otherCompanyId}, ${otherIssueId}, ${otherRunId},
${otherContractId}, ${otherResultId}, 'run_terminal', ${otherCompanyId},
'in_progress', 0, 'policy-v1',
${JSON.stringify({ disposition: "done" })}::jsonb, 'other-assessment-input-sha'
)
`;
await expect(verifySql`
INSERT INTO status_decisions (
company_id, issue_id, run_id, assessment_id, decision_version,
policy_version, from_status, to_status, reason_code,
decision_json, decision_digest
) VALUES (
${companyId}, ${issueId}, ${runId}, ${otherAssessmentId}, 1,
'policy-v1', 'in_progress', 'done', 'native_result_accepted',
${JSON.stringify({ toStatus: "done" })}::jsonb, 'cross-company-decision-sha'
)
`).rejects.toThrow(/status_decisions_assessment_owner_fk/);
await verifySql`
INSERT INTO status_decisions (
id, company_id, issue_id, run_id, assessment_id, decision_version,
policy_version, from_status, to_status, reason_code,
decision_json, decision_digest
) VALUES (
${decisionId}, ${companyId}, ${issueId}, ${runId}, ${assessmentId}, 1,
'policy-v1', 'in_progress', 'done', 'native_result_accepted',
${JSON.stringify({ toStatus: "done" })}::jsonb, 'decision-sha'
)
`;
await verifySql`
INSERT INTO status_decisions (
id, company_id, issue_id, run_id, assessment_id, decision_version,
policy_version, from_status, to_status, reason_code,
decision_json, decision_digest
) VALUES (
${otherDecisionId}, ${otherCompanyId}, ${otherIssueId}, ${otherRunId},
${otherAssessmentId}, 1, 'policy-v1', 'in_progress', 'done',
'native_result_accepted', ${JSON.stringify({ toStatus: "done" })}::jsonb,
'other-decision-sha'
)
`;
await expect(verifySql`
INSERT INTO status_decision_effects (
company_id, issue_id, decision_id, ordinal, effect_kind,
target_type, idempotency_key, payload
) VALUES (
${companyId}, ${issueId}, ${otherDecisionId}, 0, 'update_issue_status',
'issue', 'cross-company-decision-effect', ${JSON.stringify({ status: "done" })}::jsonb
)
`).rejects.toThrow(/status_decision_effects_decision_owner_fk/);
await verifySql`
INSERT INTO status_decision_effects (
company_id, issue_id, decision_id, ordinal, effect_kind,
target_type, idempotency_key, payload
) VALUES (
${companyId}, ${issueId}, ${decisionId}, 0, 'update_issue_status',
'issue', 'decision-effect-1', ${JSON.stringify({ status: "done" })}::jsonb
)
`;
await expect(verifySql`
INSERT INTO native_run_finalizations (
run_id, company_id, issue_id, phase, result_id, assessment_id, decision_id
) VALUES (
${runId}, ${companyId}, ${issueId}, 'committed', ${resultId},
${assessmentId}, ${otherDecisionId}
)
`).rejects.toThrow(/native_run_finalizations_decision_owner_fk/);
await verifySql`
INSERT INTO native_run_finalizations (
run_id, company_id, issue_id, phase, result_id, assessment_id, decision_id
) VALUES (
${runId}, ${companyId}, ${issueId}, 'committed', ${resultId}, ${assessmentId}, ${decisionId}
)
`;
await verifySql`UPDATE issues SET title = 'Renamed legacy issue' WHERE id = ${issueId}`;
await verifySql`UPDATE issues SET status = 'done' WHERE id = ${issueId}`;
const issues = await verifySql.unsafe<{ status: string; status_version: string }[]>(`
SELECT status, status_version
FROM issues
WHERE id = '${issueId}'
`);
expect(issues.map((issue) => ({
status: issue.status,
statusVersion: Number(issue.status_version),
}))).toEqual([{ status: "done", statusVersion: 1 }]);
await verifySql`DELETE FROM "drizzle"."__drizzle_migrations" WHERE "hash" = ${nativePersistenceHash}`;
} finally {
await verifySql.end();
}
await expect(applyPendingMigrations(connectionString)).resolves.toBeUndefined();
await expect(inspectMigrations(connectionString)).resolves.toMatchObject({
status: "upToDate",
});
const replaySql = postgres(connectionString, { max: 1, onnotice: () => {} });
try {
const replayed = await replaySql.unsafe<{
status_version: string;
next_event_seq: string;
trigger_count: number;
finalization_count: number;
}[]>(`
SELECT
issue.status_version,
run.next_event_seq,
(
SELECT count(*)::int
FROM pg_trigger
WHERE tgname = 'paperclip_issue_status_version_trigger'
AND NOT tgisinternal
) AS trigger_count,
(
SELECT count(*)::int
FROM native_run_finalizations
WHERE run_id = '${runId}'
) AS finalization_count
FROM issues issue
CROSS JOIN heartbeat_runs run
WHERE issue.id = '${issueId}' AND run.id = '${runId}'
`);
expect(replayed.map((row) => ({
statusVersion: Number(row.status_version),
nextEventSeq: Number(row.next_event_seq),
triggerCount: row.trigger_count,
finalizationCount: row.finalization_count,
}))).toEqual([{
statusVersion: 1,
nextEventSeq: 10,
triggerCount: 1,
finalizationCount: 1,
}]);
} finally {
await replaySql.end();
}
},
60_000,
);
});
+110 -1
View File
@@ -411,6 +411,25 @@ async function columnExists(
return rows[0]?.exists ?? false;
}
async function columnHasDataType(
sql: ReturnType<typeof postgres>,
tableName: string,
columnName: string,
dataType: string,
): Promise<boolean> {
const rows = await sql<{ dataType: string; udtName: string }[]>`
SELECT data_type AS "dataType", udt_name AS "udtName"
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = ${tableName}
AND column_name = ${columnName}
`;
const expected = dataType.toLowerCase();
return rows.some((row) => (
row.dataType.toLowerCase() === expected || row.udtName.toLowerCase() === expected
));
}
async function indexExists(
sql: ReturnType<typeof postgres>,
indexName: string,
@@ -444,11 +463,65 @@ async function constraintExists(
return rows[0]?.exists ?? false;
}
async function functionExists(
sql: ReturnType<typeof postgres>,
functionName: string,
): Promise<boolean> {
const rows = await sql<{ exists: boolean }[]>`
SELECT EXISTS (
SELECT 1
FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
WHERE n.nspname = 'public'
AND p.proname = ${functionName}
) AS exists
`;
return rows[0]?.exists ?? false;
}
async function triggerExists(
sql: ReturnType<typeof postgres>,
triggerName: string,
): Promise<boolean> {
const rows = await sql<{ exists: boolean }[]>`
SELECT EXISTS (
SELECT 1
FROM pg_trigger t
JOIN pg_class c ON c.oid = t.tgrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public'
AND t.tgname = ${triggerName}
AND NOT t.tgisinternal
) AS exists
`;
return rows[0]?.exists ?? false;
}
async function heartbeatNextEventSequencesAreCurrent(
sql: ReturnType<typeof postgres>,
): Promise<boolean> {
const rows = await sql<{ current: boolean }[]>`
SELECT NOT EXISTS (
SELECT 1
FROM heartbeat_runs run
WHERE run.next_event_seq IS DISTINCT FROM COALESCE((
SELECT max(event.seq) + 1
FROM heartbeat_run_events event
WHERE event.run_id = run.id
), 1)
) AS current
`;
return rows[0]?.current ?? false;
}
async function migrationStatementAlreadyApplied(
sql: ReturnType<typeof postgres>,
statement: string,
): Promise<boolean> {
const normalized = statement.replace(/\s+/g, " ").trim();
const normalized = statement
.replace(/^\s*--.*$/gm, "")
.replace(/\s+/g, " ")
.trim();
const createTableMatch = normalized.match(/^CREATE TABLE(?: IF NOT EXISTS)? "([^"]+)"/i);
if (createTableMatch) {
@@ -462,6 +535,18 @@ async function migrationStatementAlreadyApplied(
return columnExists(sql, addColumnMatch[1], addColumnMatch[2]);
}
const alterColumnTypeMatch = normalized.match(
/^ALTER TABLE "([^"]+)" ALTER COLUMN "([^"]+)" SET DATA TYPE ([A-Za-z0-9_]+)/i,
);
if (alterColumnTypeMatch) {
return columnHasDataType(
sql,
alterColumnTypeMatch[1],
alterColumnTypeMatch[2],
alterColumnTypeMatch[3],
);
}
const createIndexMatch = normalized.match(/^CREATE (?:UNIQUE )?INDEX(?: IF NOT EXISTS)? "([^"]+)"/i);
if (createIndexMatch) {
return indexExists(sql, createIndexMatch[1]);
@@ -472,6 +557,30 @@ async function migrationStatementAlreadyApplied(
return constraintExists(sql, addConstraintMatch[2]);
}
const createFunctionMatch = normalized.match(
/^CREATE OR REPLACE FUNCTION "?([A-Za-z_][A-Za-z0-9_]*)"?\s*\(/i,
);
if (createFunctionMatch) {
return functionExists(sql, createFunctionMatch[1]);
}
const createTriggerMatch = normalized.match(
/^CREATE TRIGGER "?([A-Za-z_][A-Za-z0-9_]*)"?/i,
);
if (createTriggerMatch) {
return triggerExists(sql, createTriggerMatch[1]);
}
// This native-runner cursor backfill has a persistent postcondition. Verify it
// instead of replaying it when a restored database is missing only the
// migration-history row.
if (
normalized.startsWith('UPDATE "heartbeat_runs" AS run')
&& normalized.includes('SET "next_event_seq" = COALESCE')
) {
return heartbeatNextEventSequencesAreCurrent(sql);
}
// If we cannot reason about a statement safely, require manual migration.
return false;
}
@@ -0,0 +1,410 @@
CREATE TABLE IF NOT EXISTS "completion_contracts" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"company_id" uuid NOT NULL,
"issue_id" uuid NOT NULL,
"revision" integer NOT NULL,
"schema_version" text NOT NULL,
"policy_version" text NOT NULL,
"risk" text NOT NULL,
"completion_authority" text NOT NULL,
"incomplete_criteria_policy" text NOT NULL,
"contract_json" jsonb NOT NULL,
"canonical_sha256" text NOT NULL,
"created_by_actor_type" text NOT NULL,
"created_by_actor_id" text NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"supersedes_contract_id" uuid
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "native_run_finalizations" (
"run_id" uuid PRIMARY KEY NOT NULL,
"company_id" uuid NOT NULL,
"issue_id" uuid NOT NULL,
"phase" text NOT NULL,
"attempt" integer DEFAULT 0 NOT NULL,
"lease_owner" text,
"lease_expires_at" timestamp with time zone,
"result_id" uuid,
"assessment_id" uuid,
"decision_id" uuid,
"failure_code" text,
"failure_detail" jsonb,
"next_attempt_at" timestamp with time zone,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "native_run_results" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"company_id" uuid NOT NULL,
"issue_id" uuid NOT NULL,
"run_id" uuid NOT NULL,
"turn_id" text,
"completion_contract_id" uuid NOT NULL,
"caller_result_id" text,
"caller_dedupe_key" text,
"server_fingerprint" text NOT NULL,
"schema_status" text NOT NULL,
"rejection_code" text,
"result_json" jsonb NOT NULL,
"canonical_sha256" text NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "status_decision_effects" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"company_id" uuid NOT NULL,
"issue_id" uuid NOT NULL,
"decision_id" uuid NOT NULL,
"ordinal" integer NOT NULL,
"effect_kind" text NOT NULL,
"target_type" text NOT NULL,
"target_id" text,
"idempotency_key" text NOT NULL,
"payload" jsonb NOT NULL,
"delivery_state" text DEFAULT 'pending' NOT NULL,
"attempt_count" integer DEFAULT 0 NOT NULL,
"next_attempt_at" timestamp with time zone,
"last_error" text,
"delivered_at" timestamp with time zone,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "status_decisions" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"company_id" uuid NOT NULL,
"issue_id" uuid NOT NULL,
"run_id" uuid NOT NULL,
"assessment_id" uuid NOT NULL,
"decision_version" bigint NOT NULL,
"policy_version" text NOT NULL,
"from_status" text NOT NULL,
"to_status" text NOT NULL,
"reason_code" text NOT NULL,
"decision_json" jsonb NOT NULL,
"decision_digest" text NOT NULL,
"application_state" text DEFAULT 'proposed' NOT NULL,
"supersedes_decision_id" uuid,
"applied_at" timestamp with time zone,
"created_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "work_assessments" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"company_id" uuid NOT NULL,
"issue_id" uuid NOT NULL,
"run_id" uuid NOT NULL,
"turn_id" text,
"contract_id" uuid NOT NULL,
"result_id" uuid NOT NULL,
"trigger_kind" text NOT NULL,
"trigger_ref" text,
"trigger_capability" text,
"trigger_actor_company_id" uuid NOT NULL,
"prior_issue_status" text NOT NULL,
"prior_status_version" bigint NOT NULL,
"prior_decision_id" uuid,
"policy_version" text NOT NULL,
"assessment_json" jsonb NOT NULL,
"input_digest" text NOT NULL,
"supersedes_assessment_id" uuid,
"created_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
ALTER TABLE "heartbeat_run_events" ALTER COLUMN "seq" SET DATA TYPE bigint;--> statement-breakpoint
ALTER TABLE "heartbeat_run_events" ADD COLUMN IF NOT EXISTS "source_instance_id" text;--> statement-breakpoint
ALTER TABLE "heartbeat_run_events" ADD COLUMN IF NOT EXISTS "source_event_id" text;--> statement-breakpoint
ALTER TABLE "heartbeat_run_events" ADD COLUMN IF NOT EXISTS "source_seq" bigint;--> statement-breakpoint
ALTER TABLE "heartbeat_run_events" ADD COLUMN IF NOT EXISTS "source_payload_sha256" text;--> statement-breakpoint
ALTER TABLE "heartbeat_run_events" ADD COLUMN IF NOT EXISTS "protocol_schema_version" integer;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "runtime_mode" text DEFAULT 'legacy' NOT NULL;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "runtime_mode_resolver_version" text;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "runtime_mode_reason" text;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "runtime_mode_resolved_at" timestamp with time zone;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "runner_profile_json" jsonb;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "runner_instance_id" uuid;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "native_session_id" uuid;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "native_issue_id" uuid;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "driver_kind" text;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "driver_version" text;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "completion_contract_id" uuid;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "completion_contract_sha256" text;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "next_event_seq" bigint DEFAULT 1 NOT NULL;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "native_phase" text;--> statement-breakpoint
ALTER TABLE "heartbeat_runs" ADD COLUMN IF NOT EXISTS "native_phase_updated_at" timestamp with time zone;--> statement-breakpoint
ALTER TABLE "status_decisions" ADD COLUMN IF NOT EXISTS "run_id" uuid;--> statement-breakpoint
ALTER TABLE "status_decisions" ALTER COLUMN "run_id" SET NOT NULL;--> statement-breakpoint
ALTER TABLE "issues" ADD COLUMN IF NOT EXISTS "status_version" bigint DEFAULT 0 NOT NULL;--> statement-breakpoint
ALTER TABLE "issues" ADD COLUMN IF NOT EXISTS "last_status_decision_id" uuid;--> statement-breakpoint
UPDATE "heartbeat_runs" AS run
SET "next_event_seq" = COALESCE((
SELECT max(event."seq") + 1
FROM "heartbeat_run_events" AS event
WHERE event."run_id" = run."id"
), 1);--> statement-breakpoint
CREATE OR REPLACE FUNCTION paperclip_bump_issue_status_version()
RETURNS trigger AS $$
BEGIN
IF NEW."status" IS DISTINCT FROM OLD."status" THEN
NEW."status_version" := OLD."status_version" + 1;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;--> statement-breakpoint
DROP TRIGGER IF EXISTS paperclip_issue_status_version_trigger ON "issues";--> statement-breakpoint
CREATE TRIGGER paperclip_issue_status_version_trigger
BEFORE UPDATE OF "status" ON "issues"
FOR EACH ROW EXECUTE FUNCTION paperclip_bump_issue_status_version();--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "completion_contracts" ADD CONSTRAINT "completion_contracts_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "completion_contracts" ADD CONSTRAINT "completion_contracts_issue_id_issues_id_fk" FOREIGN KEY ("issue_id") REFERENCES "public"."issues"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_run_id_heartbeat_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."heartbeat_runs"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_issue_id_issues_id_fk" FOREIGN KEY ("issue_id") REFERENCES "public"."issues"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_result_id_native_run_results_id_fk" FOREIGN KEY ("result_id") REFERENCES "public"."native_run_results"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_results" ADD CONSTRAINT "native_run_results_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_results" ADD CONSTRAINT "native_run_results_issue_id_issues_id_fk" FOREIGN KEY ("issue_id") REFERENCES "public"."issues"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_results" ADD CONSTRAINT "native_run_results_run_id_heartbeat_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."heartbeat_runs"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_results" ADD CONSTRAINT "native_run_results_completion_contract_id_completion_contracts_id_fk" FOREIGN KEY ("completion_contract_id") REFERENCES "public"."completion_contracts"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decision_effects" ADD CONSTRAINT "status_decision_effects_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decision_effects" ADD CONSTRAINT "status_decision_effects_issue_id_issues_id_fk" FOREIGN KEY ("issue_id") REFERENCES "public"."issues"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decision_effects" ADD CONSTRAINT "status_decision_effects_decision_id_status_decisions_id_fk" FOREIGN KEY ("decision_id") REFERENCES "public"."status_decisions"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decisions" ADD CONSTRAINT "status_decisions_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decisions" ADD CONSTRAINT "status_decisions_issue_id_issues_id_fk" FOREIGN KEY ("issue_id") REFERENCES "public"."issues"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decisions" ADD CONSTRAINT "status_decisions_assessment_id_work_assessments_id_fk" FOREIGN KEY ("assessment_id") REFERENCES "public"."work_assessments"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_issue_id_issues_id_fk" FOREIGN KEY ("issue_id") REFERENCES "public"."issues"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_run_id_heartbeat_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."heartbeat_runs"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_contract_id_completion_contracts_id_fk" FOREIGN KEY ("contract_id") REFERENCES "public"."completion_contracts"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_result_id_native_run_results_id_fk" FOREIGN KEY ("result_id") REFERENCES "public"."native_run_results"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_trigger_actor_company_id_companies_id_fk" FOREIGN KEY ("trigger_actor_company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "completion_contracts_issue_revision_uq" ON "completion_contracts" USING btree ("issue_id","revision");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "completion_contracts_issue_hash_uq" ON "completion_contracts" USING btree ("issue_id","canonical_sha256");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "native_run_results_run_fingerprint_uq" ON "native_run_results" USING btree ("run_id","server_fingerprint");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "native_run_results_run_caller_result_uq" ON "native_run_results" USING btree ("run_id","caller_result_id");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "native_run_results_run_caller_dedupe_uq" ON "native_run_results" USING btree ("run_id","caller_dedupe_key");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "status_decision_effects_decision_ordinal_uq" ON "status_decision_effects" USING btree ("decision_id","ordinal");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "status_decision_effects_company_idempotency_uq" ON "status_decision_effects" USING btree ("company_id","idempotency_key");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "status_decisions_company_issue_version_uq" ON "status_decisions" USING btree ("company_id","issue_id","decision_version");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "status_decisions_company_assessment_uq" ON "status_decisions" USING btree ("company_id","assessment_id");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "status_decisions_company_issue_digest_uq" ON "status_decisions" USING btree ("company_id","issue_id","decision_digest");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "work_assessments_company_issue_input_uq" ON "work_assessments" USING btree ("company_id","issue_id","input_digest");--> statement-breakpoint
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: nullable native source IDs mean historical rows need no backfill and the invariant must commit atomically with the new columns.
CREATE UNIQUE INDEX IF NOT EXISTS "heartbeat_run_events_run_source_event_uq" ON "heartbeat_run_events" USING btree ("run_id","source_event_id") WHERE "heartbeat_run_events"."source_event_id" is not null;--> statement-breakpoint
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: nullable native source sequence fields mean historical rows need no backfill and the invariant must commit atomically with the new columns.
CREATE UNIQUE INDEX IF NOT EXISTS "heartbeat_run_events_run_source_seq_uq" ON "heartbeat_run_events" USING btree ("run_id","source_instance_id","source_seq") WHERE "heartbeat_run_events"."source_instance_id" is not null and "heartbeat_run_events"."source_seq" is not null;
--> statement-breakpoint
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: the primary key already makes this ownership tuple unique; this supporting index only enables composite foreign keys.
CREATE UNIQUE INDEX IF NOT EXISTS "issues_company_id_uq" ON "issues" USING btree ("company_id","id");--> statement-breakpoint
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: the primary key already makes this ownership tuple unique; this supporting index only enables composite foreign keys.
CREATE UNIQUE INDEX IF NOT EXISTS "heartbeat_runs_company_native_issue_id_uq" ON "heartbeat_runs" USING btree ("company_id","native_issue_id","id");--> statement-breakpoint
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: the primary key already makes this ownership tuple unique; this supporting index only enables composite foreign keys.
CREATE UNIQUE INDEX IF NOT EXISTS "heartbeat_runs_company_native_issue_contract_id_uq" ON "heartbeat_runs" USING btree ("company_id","native_issue_id","id","completion_contract_id");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "completion_contracts_company_issue_id_uq" ON "completion_contracts" USING btree ("company_id","issue_id","id");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "native_run_results_company_issue_run_id_uq" ON "native_run_results" USING btree ("company_id","issue_id","run_id","id");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "work_assessments_company_issue_run_id_uq" ON "work_assessments" USING btree ("company_id","issue_id","run_id","id");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "status_decisions_company_issue_id_uq" ON "status_decisions" USING btree ("company_id","issue_id","id");--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "status_decisions_company_issue_run_assessment_id_uq" ON "status_decisions" USING btree ("company_id","issue_id","run_id","assessment_id","id");--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "completion_contracts" ADD CONSTRAINT "completion_contracts_issue_company_fk" FOREIGN KEY ("company_id","issue_id") REFERENCES "public"."issues"("company_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "completion_contracts" ADD CONSTRAINT "completion_contracts_supersedes_owner_fk" FOREIGN KEY ("company_id","issue_id","supersedes_contract_id") REFERENCES "public"."completion_contracts"("company_id","issue_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_results" ADD CONSTRAINT "native_run_results_issue_company_fk" FOREIGN KEY ("company_id","issue_id") REFERENCES "public"."issues"("company_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_results" ADD CONSTRAINT "native_run_results_run_contract_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id","completion_contract_id") REFERENCES "public"."heartbeat_runs"("company_id","native_issue_id","id","completion_contract_id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_results" ADD CONSTRAINT "native_run_results_completion_contract_owner_fk" FOREIGN KEY ("company_id","issue_id","completion_contract_id") REFERENCES "public"."completion_contracts"("company_id","issue_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_issue_company_fk" FOREIGN KEY ("company_id","issue_id") REFERENCES "public"."issues"("company_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_run_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id") REFERENCES "public"."heartbeat_runs"("company_id","native_issue_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_contract_owner_fk" FOREIGN KEY ("company_id","issue_id","contract_id") REFERENCES "public"."completion_contracts"("company_id","issue_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_result_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id","result_id") REFERENCES "public"."native_run_results"("company_id","issue_id","run_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_supersedes_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id","supersedes_assessment_id") REFERENCES "public"."work_assessments"("company_id","issue_id","run_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decisions" ADD CONSTRAINT "status_decisions_issue_company_fk" FOREIGN KEY ("company_id","issue_id") REFERENCES "public"."issues"("company_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decisions" ADD CONSTRAINT "status_decisions_assessment_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id","assessment_id") REFERENCES "public"."work_assessments"("company_id","issue_id","run_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decisions" ADD CONSTRAINT "status_decisions_supersedes_owner_fk" FOREIGN KEY ("company_id","issue_id","supersedes_decision_id") REFERENCES "public"."status_decisions"("company_id","issue_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decision_effects" ADD CONSTRAINT "status_decision_effects_issue_company_fk" FOREIGN KEY ("company_id","issue_id") REFERENCES "public"."issues"("company_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "status_decision_effects" ADD CONSTRAINT "status_decision_effects_decision_owner_fk" FOREIGN KEY ("company_id","issue_id","decision_id") REFERENCES "public"."status_decisions"("company_id","issue_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_issue_company_fk" FOREIGN KEY ("company_id","issue_id") REFERENCES "public"."issues"("company_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_run_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id") REFERENCES "public"."heartbeat_runs"("company_id","native_issue_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_result_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id","result_id") REFERENCES "public"."native_run_results"("company_id","issue_id","run_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_assessment_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id","assessment_id") REFERENCES "public"."work_assessments"("company_id","issue_id","run_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_decision_owner_fk" FOREIGN KEY ("company_id","issue_id","run_id","assessment_id","decision_id") REFERENCES "public"."status_decisions"("company_id","issue_id","run_id","assessment_id","id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_assessment_requires_result_check" CHECK ("assessment_id" is null or "result_id" is not null);
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "native_run_finalizations" ADD CONSTRAINT "native_run_finalizations_decision_requires_assessment_check" CHECK ("decision_id" is null or "assessment_id" is not null);
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "work_assessments" ADD CONSTRAINT "work_assessments_trigger_actor_company_check" CHECK ("trigger_actor_company_id" = "company_id");
EXCEPTION
WHEN duplicate_object THEN NULL;
END $$;
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+10 -3
View File
@@ -1580,9 +1580,16 @@
{
"idx": 227,
"version": "7",
"when": 1787470797345,
"tag": "0227_zippy_blockbuster",
"when": 1787668790923,
"tag": "0227_modern_pandemic",
"breakpoints": true
},
{
"idx": 228,
"version": "7",
"when": 1787685849681,
"tag": "0228_zippy_blockbuster",
"breakpoints": true
}
]
}
}
@@ -0,0 +1,59 @@
import {
foreignKey,
pgTable,
uuid,
text,
timestamp,
integer,
jsonb,
unique,
uniqueIndex,
} from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
export const completionContracts = pgTable(
"completion_contracts",
{
id: uuid("id").primaryKey().defaultRandom(),
companyId: uuid("company_id").notNull().references(() => companies.id),
issueId: uuid("issue_id").notNull(),
revision: integer("revision").notNull(),
schemaVersion: text("schema_version").notNull(),
policyVersion: text("policy_version").notNull(),
risk: text("risk").notNull(),
completionAuthority: text("completion_authority").notNull(),
incompleteCriteriaPolicy: text("incomplete_criteria_policy").notNull(),
contractJson: jsonb("contract_json").$type<Record<string, unknown>>().notNull(),
canonicalSha256: text("canonical_sha256").notNull(),
createdByActorType: text("created_by_actor_type").notNull(),
createdByActorId: text("created_by_actor_id").notNull(),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
supersedesContractId: uuid("supersedes_contract_id"),
},
(table) => ({
companyIssueIdUq: unique("completion_contracts_company_issue_id_uq").on(
table.companyId,
table.issueId,
table.id,
),
issueCompanyFk: foreignKey({
columns: [table.companyId, table.issueId],
foreignColumns: [issues.companyId, issues.id],
name: "completion_contracts_issue_company_fk",
}),
supersedesOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.supersedesContractId],
foreignColumns: [table.companyId, table.issueId, table.id],
name: "completion_contracts_supersedes_owner_fk",
}),
issueRevisionUq: uniqueIndex("completion_contracts_issue_revision_uq").on(
table.issueId,
table.revision,
),
issueHashUq: uniqueIndex("completion_contracts_issue_hash_uq").on(
table.issueId,
table.canonicalSha256,
),
}),
);
+25 -3
View File
@@ -1,4 +1,16 @@
import { pgTable, uuid, text, timestamp, integer, jsonb, index, bigserial } from "drizzle-orm/pg-core";
import { sql } from "drizzle-orm";
import {
pgTable,
uuid,
text,
timestamp,
integer,
jsonb,
index,
bigserial,
bigint,
uniqueIndex,
} from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { agents } from "./agents.js";
import { heartbeatRuns } from "./heartbeat_runs.js";
@@ -10,19 +22,29 @@ export const heartbeatRunEvents = pgTable(
companyId: uuid("company_id").notNull().references(() => companies.id),
runId: uuid("run_id").notNull().references(() => heartbeatRuns.id),
agentId: uuid("agent_id").notNull().references(() => agents.id),
seq: integer("seq").notNull(),
seq: bigint("seq", { mode: "number" }).notNull(),
eventType: text("event_type").notNull(),
stream: text("stream"),
level: text("level"),
color: text("color"),
message: text("message"),
payload: jsonb("payload").$type<Record<string, unknown>>(),
sourceInstanceId: text("source_instance_id"),
sourceEventId: text("source_event_id"),
sourceSeq: bigint("source_seq", { mode: "number" }),
sourcePayloadSha256: text("source_payload_sha256"),
protocolSchemaVersion: integer("protocol_schema_version"),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
runSeqIdx: index("heartbeat_run_events_run_seq_idx").on(table.runId, table.seq),
runSourceEventUq: uniqueIndex("heartbeat_run_events_run_source_event_uq")
.on(table.runId, table.sourceEventId)
.where(sql`${table.sourceEventId} is not null`),
runSourceSeqUq: uniqueIndex("heartbeat_run_events_run_source_seq_uq")
.on(table.runId, table.sourceInstanceId, table.sourceSeq)
.where(sql`${table.sourceInstanceId} is not null and ${table.sourceSeq} is not null`),
companyRunIdx: index("heartbeat_run_events_company_run_idx").on(table.companyId, table.runId),
companyCreatedIdx: index("heartbeat_run_events_company_created_idx").on(table.companyId, table.createdAt),
}),
);
+41 -1
View File
@@ -1,5 +1,17 @@
import { sql } from "drizzle-orm";
import { type AnyPgColumn, pgTable, uuid, text, timestamp, jsonb, index, integer, bigint, boolean } from "drizzle-orm/pg-core";
import {
type AnyPgColumn,
pgTable,
uuid,
text,
timestamp,
jsonb,
index,
integer,
bigint,
boolean,
unique,
} from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { agents } from "./agents.js";
import { agentWakeupRequests } from "./agent_wakeup_requests.js";
@@ -22,6 +34,21 @@ export const heartbeatRuns = pgTable(
signal: text("signal"),
usageJson: jsonb("usage_json").$type<Record<string, unknown>>(),
resultJson: jsonb("result_json").$type<Record<string, unknown>>(),
runtimeMode: text("runtime_mode").notNull().default("legacy"),
runtimeModeResolverVersion: text("runtime_mode_resolver_version"),
runtimeModeReason: text("runtime_mode_reason"),
runtimeModeResolvedAt: timestamp("runtime_mode_resolved_at", { withTimezone: true }),
runnerProfileJson: jsonb("runner_profile_json").$type<Record<string, unknown>>(),
runnerInstanceId: uuid("runner_instance_id"),
nativeSessionId: uuid("native_session_id"),
nativeIssueId: uuid("native_issue_id"),
driverKind: text("driver_kind"),
driverVersion: text("driver_version"),
completionContractId: uuid("completion_contract_id"),
completionContractSha256: text("completion_contract_sha256"),
nextEventSeq: bigint("next_event_seq", { mode: "number" }).notNull().default(1),
nativePhase: text("native_phase"),
nativePhaseUpdatedAt: timestamp("native_phase_updated_at", { withTimezone: true }),
sessionIdBefore: text("session_id_before"),
sessionIdAfter: text("session_id_after"),
logStore: text("log_store"),
@@ -60,6 +87,19 @@ export const heartbeatRuns = pgTable(
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
companyNativeIssueRunUq: unique("heartbeat_runs_company_native_issue_id_uq").on(
table.companyId,
table.nativeIssueId,
table.id,
),
companyNativeIssueRunContractUq: unique(
"heartbeat_runs_company_native_issue_contract_id_uq",
).on(
table.companyId,
table.nativeIssueId,
table.id,
table.completionContractId,
),
companyAgentStartedIdx: index("heartbeat_runs_company_agent_started_idx").on(
table.companyId,
table.agentId,
+6
View File
@@ -111,6 +111,12 @@ export { documentAnnotationComments } from "./document_annotation_comments.js";
export { documentAnnotationAnchorSnapshots } from "./document_annotation_anchor_snapshots.js";
export { heartbeatRuns } from "./heartbeat_runs.js";
export { heartbeatRunEvents } from "./heartbeat_run_events.js";
export { completionContracts } from "./completion_contracts.js";
export { nativeRunResults } from "./native_run_results.js";
export { nativeRunFinalizations } from "./native_run_finalizations.js";
export { workAssessments } from "./work_assessments.js";
export { statusDecisions } from "./status_decisions.js";
export { statusDecisionEffects } from "./status_decision_effects.js";
export { heartbeatRunWatchdogDecisions } from "./heartbeat_run_watchdog_decisions.js";
export { smokeRuns, smokeRunSteps } from "./smoke_lab.js";
export { costEvents } from "./cost_events.js";
+5
View File
@@ -9,6 +9,8 @@ import {
jsonb,
index,
uniqueIndex,
unique,
bigint,
} from "drizzle-orm/pg-core";
import { agents } from "./agents.js";
import { projects } from "./projects.js";
@@ -31,6 +33,8 @@ export const issues = pgTable(
title: text("title").notNull(),
description: text("description"),
status: text("status").notNull().default("backlog"),
statusVersion: bigint("status_version", { mode: "number" }).notNull().default(0),
lastStatusDecisionId: uuid("last_status_decision_id"),
workMode: text("work_mode").notNull().default("standard"),
harnessKind: text("harness_kind"),
priority: text("priority").notNull().default("medium"),
@@ -78,6 +82,7 @@ export const issues = pgTable(
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
companyIdUq: unique("issues_company_id_uq").on(table.companyId, table.id),
companyStatusIdx: index("issues_company_status_idx").on(table.companyId, table.status),
companyHarnessKindIdx: index("issues_company_harness_kind_idx").on(table.companyId, table.harnessKind),
assigneeStatusIdx: index("issues_company_assignee_status_idx").on(
@@ -0,0 +1,95 @@
import { sql } from "drizzle-orm";
import {
check,
foreignKey,
pgTable,
uuid,
text,
timestamp,
integer,
jsonb,
} from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
import { heartbeatRuns } from "./heartbeat_runs.js";
import { nativeRunResults } from "./native_run_results.js";
import { statusDecisions } from "./status_decisions.js";
import { workAssessments } from "./work_assessments.js";
export const nativeRunFinalizations = pgTable(
"native_run_finalizations",
{
runId: uuid("run_id").primaryKey(),
companyId: uuid("company_id").notNull().references(() => companies.id),
issueId: uuid("issue_id").notNull(),
phase: text("phase").notNull(),
attempt: integer("attempt").notNull().default(0),
leaseOwner: text("lease_owner"),
leaseExpiresAt: timestamp("lease_expires_at", { withTimezone: true }),
resultId: uuid("result_id"),
assessmentId: uuid("assessment_id"),
decisionId: uuid("decision_id"),
failureCode: text("failure_code"),
failureDetail: jsonb("failure_detail").$type<Record<string, unknown>>(),
nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
issueCompanyFk: foreignKey({
columns: [table.companyId, table.issueId],
foreignColumns: [issues.companyId, issues.id],
name: "native_run_finalizations_issue_company_fk",
}),
runOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.runId],
foreignColumns: [heartbeatRuns.companyId, heartbeatRuns.nativeIssueId, heartbeatRuns.id],
name: "native_run_finalizations_run_owner_fk",
}),
resultOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.runId, table.resultId],
foreignColumns: [
nativeRunResults.companyId,
nativeRunResults.issueId,
nativeRunResults.runId,
nativeRunResults.id,
],
name: "native_run_finalizations_result_owner_fk",
}),
assessmentOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.runId, table.assessmentId],
foreignColumns: [
workAssessments.companyId,
workAssessments.issueId,
workAssessments.runId,
workAssessments.id,
],
name: "native_run_finalizations_assessment_owner_fk",
}),
decisionOwnerFk: foreignKey({
columns: [
table.companyId,
table.issueId,
table.runId,
table.assessmentId,
table.decisionId,
],
foreignColumns: [
statusDecisions.companyId,
statusDecisions.issueId,
statusDecisions.runId,
statusDecisions.assessmentId,
statusDecisions.id,
],
name: "native_run_finalizations_decision_owner_fk",
}),
assessmentRequiresResultCheck: check(
"native_run_finalizations_assessment_requires_result_check",
sql`${table.assessmentId} is null or ${table.resultId} is not null`,
),
decisionRequiresAssessmentCheck: check(
"native_run_finalizations_decision_requires_assessment_check",
sql`${table.decisionId} is null or ${table.assessmentId} is not null`,
),
}),
);
@@ -0,0 +1,83 @@
import {
foreignKey,
pgTable,
uuid,
text,
timestamp,
jsonb,
unique,
uniqueIndex,
} from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
import { heartbeatRuns } from "./heartbeat_runs.js";
import { completionContracts } from "./completion_contracts.js";
export const nativeRunResults = pgTable(
"native_run_results",
{
id: uuid("id").primaryKey().defaultRandom(),
companyId: uuid("company_id").notNull().references(() => companies.id),
issueId: uuid("issue_id").notNull(),
runId: uuid("run_id").notNull(),
turnId: text("turn_id"),
completionContractId: uuid("completion_contract_id").notNull(),
callerResultId: text("caller_result_id"),
callerDedupeKey: text("caller_dedupe_key"),
serverFingerprint: text("server_fingerprint").notNull(),
schemaStatus: text("schema_status").notNull(),
rejectionCode: text("rejection_code"),
resultJson: jsonb("result_json").$type<Record<string, unknown>>().notNull(),
canonicalSha256: text("canonical_sha256").notNull(),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
companyIssueRunIdUq: unique("native_run_results_company_issue_run_id_uq").on(
table.companyId,
table.issueId,
table.runId,
table.id,
),
issueCompanyFk: foreignKey({
columns: [table.companyId, table.issueId],
foreignColumns: [issues.companyId, issues.id],
name: "native_run_results_issue_company_fk",
}),
runContractOwnerFk: foreignKey({
columns: [
table.companyId,
table.issueId,
table.runId,
table.completionContractId,
],
foreignColumns: [
heartbeatRuns.companyId,
heartbeatRuns.nativeIssueId,
heartbeatRuns.id,
heartbeatRuns.completionContractId,
],
name: "native_run_results_run_contract_owner_fk",
}),
completionContractOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.completionContractId],
foreignColumns: [
completionContracts.companyId,
completionContracts.issueId,
completionContracts.id,
],
name: "native_run_results_completion_contract_owner_fk",
}),
runFingerprintUq: uniqueIndex("native_run_results_run_fingerprint_uq").on(
table.runId,
table.serverFingerprint,
),
runCallerResultUq: uniqueIndex("native_run_results_run_caller_result_uq").on(
table.runId,
table.callerResultId,
),
runCallerDedupeUq: uniqueIndex("native_run_results_run_caller_dedupe_uq").on(
table.runId,
table.callerDedupeKey,
),
}),
);
@@ -0,0 +1,56 @@
import {
foreignKey,
pgTable,
uuid,
text,
timestamp,
integer,
jsonb,
uniqueIndex,
} from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
import { statusDecisions } from "./status_decisions.js";
export const statusDecisionEffects = pgTable(
"status_decision_effects",
{
id: uuid("id").primaryKey().defaultRandom(),
companyId: uuid("company_id").notNull().references(() => companies.id),
issueId: uuid("issue_id").notNull(),
decisionId: uuid("decision_id").notNull(),
ordinal: integer("ordinal").notNull(),
effectKind: text("effect_kind").notNull(),
targetType: text("target_type").notNull(),
targetId: text("target_id"),
idempotencyKey: text("idempotency_key").notNull(),
payload: jsonb("payload").$type<Record<string, unknown>>().notNull(),
deliveryState: text("delivery_state").notNull().default("pending"),
attemptCount: integer("attempt_count").notNull().default(0),
nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }),
lastError: text("last_error"),
deliveredAt: timestamp("delivered_at", { withTimezone: true }),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
issueCompanyFk: foreignKey({
columns: [table.companyId, table.issueId],
foreignColumns: [issues.companyId, issues.id],
name: "status_decision_effects_issue_company_fk",
}),
decisionOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.decisionId],
foreignColumns: [statusDecisions.companyId, statusDecisions.issueId, statusDecisions.id],
name: "status_decision_effects_decision_owner_fk",
}),
decisionOrdinalUq: uniqueIndex("status_decision_effects_decision_ordinal_uq").on(
table.decisionId,
table.ordinal,
),
idempotencyUq: uniqueIndex("status_decision_effects_company_idempotency_uq").on(
table.companyId,
table.idempotencyKey,
),
}),
);
@@ -0,0 +1,86 @@
import {
foreignKey,
pgTable,
uuid,
text,
timestamp,
bigint,
jsonb,
unique,
uniqueIndex,
} from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
import { workAssessments } from "./work_assessments.js";
export const statusDecisions = pgTable(
"status_decisions",
{
id: uuid("id").primaryKey().defaultRandom(),
companyId: uuid("company_id").notNull().references(() => companies.id),
issueId: uuid("issue_id").notNull(),
runId: uuid("run_id").notNull(),
assessmentId: uuid("assessment_id").notNull(),
decisionVersion: bigint("decision_version", { mode: "number" }).notNull(),
policyVersion: text("policy_version").notNull(),
fromStatus: text("from_status").notNull(),
toStatus: text("to_status").notNull(),
reasonCode: text("reason_code").notNull(),
decisionJson: jsonb("decision_json").$type<Record<string, unknown>>().notNull(),
decisionDigest: text("decision_digest").notNull(),
applicationState: text("application_state").notNull().default("proposed"),
supersedesDecisionId: uuid("supersedes_decision_id"),
appliedAt: timestamp("applied_at", { withTimezone: true }),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
companyIssueIdUq: unique("status_decisions_company_issue_id_uq").on(
table.companyId,
table.issueId,
table.id,
),
companyIssueRunAssessmentIdUq: unique(
"status_decisions_company_issue_run_assessment_id_uq",
).on(
table.companyId,
table.issueId,
table.runId,
table.assessmentId,
table.id,
),
issueCompanyFk: foreignKey({
columns: [table.companyId, table.issueId],
foreignColumns: [issues.companyId, issues.id],
name: "status_decisions_issue_company_fk",
}),
assessmentOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.runId, table.assessmentId],
foreignColumns: [
workAssessments.companyId,
workAssessments.issueId,
workAssessments.runId,
workAssessments.id,
],
name: "status_decisions_assessment_owner_fk",
}),
supersedesOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.supersedesDecisionId],
foreignColumns: [table.companyId, table.issueId, table.id],
name: "status_decisions_supersedes_owner_fk",
}),
issueVersionUq: uniqueIndex("status_decisions_company_issue_version_uq").on(
table.companyId,
table.issueId,
table.decisionVersion,
),
assessmentUq: uniqueIndex("status_decisions_company_assessment_uq").on(
table.companyId,
table.assessmentId,
),
issueDigestUq: uniqueIndex("status_decisions_company_issue_digest_uq").on(
table.companyId,
table.issueId,
table.decisionDigest,
),
}),
);
@@ -0,0 +1,94 @@
import { sql } from "drizzle-orm";
import {
check,
foreignKey,
pgTable,
uuid,
text,
timestamp,
bigint,
jsonb,
unique,
uniqueIndex,
} from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
import { heartbeatRuns } from "./heartbeat_runs.js";
import { completionContracts } from "./completion_contracts.js";
import { nativeRunResults } from "./native_run_results.js";
export const workAssessments = pgTable(
"work_assessments",
{
id: uuid("id").primaryKey().defaultRandom(),
companyId: uuid("company_id").notNull().references(() => companies.id),
issueId: uuid("issue_id").notNull(),
runId: uuid("run_id").notNull(),
turnId: text("turn_id"),
contractId: uuid("contract_id").notNull(),
resultId: uuid("result_id").notNull(),
triggerKind: text("trigger_kind").notNull(),
triggerRef: text("trigger_ref"),
triggerCapability: text("trigger_capability"),
triggerActorCompanyId: uuid("trigger_actor_company_id").notNull().references(() => companies.id),
priorIssueStatus: text("prior_issue_status").notNull(),
priorStatusVersion: bigint("prior_status_version", { mode: "number" }).notNull(),
priorDecisionId: uuid("prior_decision_id"),
policyVersion: text("policy_version").notNull(),
assessmentJson: jsonb("assessment_json").$type<Record<string, unknown>>().notNull(),
inputDigest: text("input_digest").notNull(),
supersedesAssessmentId: uuid("supersedes_assessment_id"),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
},
(table) => ({
companyIssueRunIdUq: unique("work_assessments_company_issue_run_id_uq").on(
table.companyId,
table.issueId,
table.runId,
table.id,
),
issueCompanyFk: foreignKey({
columns: [table.companyId, table.issueId],
foreignColumns: [issues.companyId, issues.id],
name: "work_assessments_issue_company_fk",
}),
runOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.runId],
foreignColumns: [heartbeatRuns.companyId, heartbeatRuns.nativeIssueId, heartbeatRuns.id],
name: "work_assessments_run_owner_fk",
}),
contractOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.contractId],
foreignColumns: [
completionContracts.companyId,
completionContracts.issueId,
completionContracts.id,
],
name: "work_assessments_contract_owner_fk",
}),
resultOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.runId, table.resultId],
foreignColumns: [
nativeRunResults.companyId,
nativeRunResults.issueId,
nativeRunResults.runId,
nativeRunResults.id,
],
name: "work_assessments_result_owner_fk",
}),
supersedesOwnerFk: foreignKey({
columns: [table.companyId, table.issueId, table.runId, table.supersedesAssessmentId],
foreignColumns: [table.companyId, table.issueId, table.runId, table.id],
name: "work_assessments_supersedes_owner_fk",
}),
triggerActorCompanyCheck: check(
"work_assessments_trigger_actor_company_check",
sql`${table.triggerActorCompanyId} = ${table.companyId}`,
),
issueInputUq: uniqueIndex("work_assessments_company_issue_input_uq").on(
table.companyId,
table.issueId,
table.inputDigest,
),
}),
);
Loaded 100 of 374 files, more files were not shown because too many files have changed in this diff. Show more