Merge branch 'codex/boat-environment-core' into codex/boat-environment

* codex/boat-environment-core:
  fix: preserve authoritative warm computers during orphan lease sweep
  fix: share Boat upload capacity before command admission
This commit is contained in:
Dotta committed 2026-10-11 03:01:30 -05:00
commit e2c390dd04
12 files changed
+344 -23

No files matched your search

@@ -58,6 +58,16 @@ export interface CommandManagedDuplexChannel {
}
export interface CommandManagedRuntimeRunner {
/**
* Optional transport-shared backpressure for stdin artifact chunks. Acquire
* before execute starts its command deadline; this grants no execution
* authority. The callback must still use the owned execute wrapper. Queue
* waiting is bounded separately, and signal cancels only a queued operation.
*/
withStdinUploadSlot?<T>(
operation: () => Promise<T>,
options: { timeoutMs: number; signal?: AbortSignal },
): Promise<T>;
/**
* True when the provider verified the concurrent-sync opt-in. A native runner
* carries the value from the effective capability snapshot
@@ -5,6 +5,7 @@ import {
costEvents,
agents,
companies,
computers,
createDb,
environmentLeases,
environments,
@@ -65,6 +66,7 @@ describeEmbeddedPostgres("heartbeat sweepOrphanedActiveLeases", () => {
await db.delete(environmentLeases);
await db.delete(costEvents);
await db.delete(heartbeatRuns);
await db.delete(computers);
await db.delete(environments);
await db.delete(agents);
await db.delete(companies);
@@ -98,7 +100,7 @@ describeEmbeddedPostgres("heartbeat sweepOrphanedActiveLeases", () => {
await db.insert(environments).values({
id: environmentId,
companyId,
name: "Fake Sandbox",
name: `Fake Sandbox ${environmentId}`,
driver: "sandbox",
status: "active",
config: { provider: "fake", image: "ubuntu:24.04" },
@@ -160,6 +162,104 @@ describeEmbeddedPostgres("heartbeat sweepOrphanedActiveLeases", () => {
const oldEnough = () => new Date(Date.now() - 60 * 60 * 1000);
async function seedWarmComputerLease() {
const scope = await seedCompanyAgentAndEnvironment();
await db.update(environments).set({ driver: "computer" }).where(eq(environments.id, scope.environmentId));
const runId = await insertHeartbeatRun({ ...scope, status: "succeeded" });
const computerId = randomUUID(), ownerId = randomUUID();
const owner = { id: ownerId, generation: 1, kind: "runner", phase: "warm", agentId: scope.agentId,
runId, admittedAt: oldEnough().toISOString(), deadline: new Date(Date.now() + 300_000).toISOString(),
absoluteDeadline: null, retirementDeadline: null, process: { bootId: randomUUID(), unitName: "test", nonce: "test", launchGeneration: 1 } };
await db.insert(computers).values({ id: computerId, companyId: scope.companyId, environmentId: scope.environmentId,
provider: "boat", providerId: randomUUID(), ledger: { status: "attached", owners: [owner], placements: {}, action: null } });
const leaseId = await insertActiveLease({ ...scope, heartbeatRunId: runId, updatedAt: oldEnough(),
provider: "boat", providerLeaseId: ownerId });
await db.update(environmentLeases).set({ metadata: { driver: "computer", agentId: scope.agentId,
computerOwner: { computerId, ownerId, generation: 1 } } }).where(eq(environmentLeases.id, leaseId));
return { ...scope, runId, computerId, ownerId, leaseId, owner };
}
it("preserves an authoritative warm computer while successful-run instruction cleanup delays lease retention", async () => {
const fixture = await seedWarmComputerLease();
const destroyRunLease = vi.fn();
const heartbeat = heartbeatService(db, { environmentRuntime: { destroyRunLease } as unknown as HeartbeatEnvironmentRuntime });
expect(await heartbeat.sweepOrphanedActiveLeases({ backoffMs: 300_000 })).toEqual({ recovered: 0 });
await heartbeat.sweepPendingCleanupLeases({ backoffMs: 0 });
expect(destroyRunLease).not.toHaveBeenCalled();
expect(await leaseRow(fixture.leaseId)).toMatchObject({ status: "active", failureReason: null });
// The normal heartbeat can finish its delayed cleanup and retain the lease.
await db.update(environmentLeases).set({ status: "retained" }).where(eq(environmentLeases.id, fixture.leaseId));
expect(await heartbeat.sweepOrphanedActiveLeases({ backoffMs: 0 })).toEqual({ recovered: 0 });
});
it.each(["expired", "retired", "foreign_run", "foreign_agent", "generation", "computer", "environment", "company", "provider_lease", "invalid_deadline", "missing_process", "invalid_process", "invalid_launch_generation"])(
"still recovers a computer lease without exact bounded warm authority: %s", async (kind) => {
const fixture = await seedWarmComputerLease();
const owner = { ...fixture.owner };
if (kind === "expired") owner.deadline = new Date(Date.now() - 60_000).toISOString();
if (kind === "retired") owner.phase = "retired";
if (kind === "foreign_run") owner.runId = randomUUID();
if (kind === "foreign_agent") owner.agentId = randomUUID();
if (kind === "generation") owner.generation += 1;
if (kind === "invalid_deadline") owner.deadline = "invalid";
if (kind === "missing_process") Object.assign(owner, { process: null });
if (kind === "invalid_process") owner.process = { ...owner.process, nonce: "" };
if (kind === "invalid_launch_generation") owner.process = { ...owner.process, launchGeneration: 0 };
await db.update(computers).set({ ledger: { status: "attached", owners: [owner] } }).where(eq(computers.id, fixture.computerId));
if (kind === "computer") await db.update(environmentLeases).set({ metadata: { driver: "computer", agentId: fixture.agentId,
computerOwner: { computerId: randomUUID(), ownerId: fixture.ownerId, generation: 1 } } }).where(eq(environmentLeases.id, fixture.leaseId));
if (kind === "provider_lease") await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, fixture.leaseId));
if (kind === "environment" || kind === "company") {
const foreign = await seedCompanyAgentAndEnvironment();
await db.update(computers).set(kind === "company" ? { companyId: foreign.companyId } : { environmentId: foreign.environmentId })
.where(eq(computers.id, fixture.computerId));
}
expect(await heartbeatService(db).sweepOrphanedActiveLeases({ backoffMs: 300_000 })).toEqual({ recovered: 1 });
expect(await leaseRow(fixture.leaseId)).toMatchObject({ status: "pending_cleanup", failureReason: "orphaned_active_lease_recovered" });
});
it("preserves a warm owner whose stopped process was relaunched within the same owner generation", async () => {
const fixture = await seedWarmComputerLease();
await db.update(computers).set({ ledger: { owners: [{ ...fixture.owner,
process: { ...fixture.owner.process, launchGeneration: 2 } }] } })
.where(eq(computers.id, fixture.computerId));
// Process launch generation is an independent counter, not an owner epoch.
expect(await heartbeatService(db).sweepOrphanedActiveLeases({ backoffMs: 300_000 })).toEqual({ recovered: 0 });
});
it("honors only the existing bounded warm shutdown grace", async () => {
const fixture = await seedWarmComputerLease();
await db.update(computers).set({ ledger: { owners: [{ ...fixture.owner, deadline: new Date(Date.now() - 10_000).toISOString() }] } })
.where(eq(computers.id, fixture.computerId));
expect(await heartbeatService(db).sweepOrphanedActiveLeases({ backoffMs: 300_000 })).toEqual({ recovered: 0 });
});
it("allows cleanup after the same warm reservation and shutdown grace expire", async () => {
const fixture = await seedWarmComputerLease();
const heartbeat = heartbeatService(db);
expect(await heartbeat.sweepOrphanedActiveLeases({ backoffMs: 0 })).toEqual({ recovered: 0 });
await db.update(computers).set({ ledger: { owners: [{ ...fixture.owner, deadline: new Date(Date.now() - 60_000).toISOString() }] } })
.where(eq(computers.id, fixture.computerId));
expect(await heartbeat.sweepOrphanedActiveLeases({ backoffMs: 0 })).toEqual({ recovered: 1 });
});
it("does not overwrite lease retention completed after the sweep selected its stale snapshot", async () => {
const fixture = await seedWarmComputerLease();
await db.update(computers).set({ ledger: { owners: [{ ...fixture.owner, phase: "retired" }] } })
.where(eq(computers.id, fixture.computerId));
const transaction = db.transaction.bind(db);
const intercept = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => {
// Sweep has selected the old active row, but heartbeat finishes retention
// before the computer-row lock/cleanup claim. The status CAS must lose.
await db.update(environmentLeases).set({ status: "retained" }).where(eq(environmentLeases.id, fixture.leaseId));
return transaction(callback, config);
});
try {
expect(await heartbeatService(db).sweepOrphanedActiveLeases({ backoffMs: 0 })).toEqual({ recovered: 0 });
expect(await leaseRow(fixture.leaseId)).toMatchObject({ status: "retained", failureReason: null });
} finally { intercept.mockRestore(); }
});
it.each([false, true])("releases an interrupted local run's bookkeeping lease (deleted environment: %s)", async (deletedEnvironment) => {
const { companyId, agentId, environmentId } = await seedCompanyAgentAndEnvironment();
await db.delete(environments).where(eq(environments.driver, "local"));
@@ -891,6 +891,34 @@ describe("Boat SSH admission", () => {
await Promise.all([...jobs, queued]);
} finally { vi.useRealTimers(); }
});
it("bounds and cancels queued upload slots without starting their callbacks", async () => {
vi.useFakeTimers();
try {
const admission = createBoatTransportAdmission();
const completions: Array<() => void> = [];
const occupied = Array.from({ length: 4 }, () => admission.withStdinUploadSlot(
() => new Promise<void>(resolve => completions.push(resolve)), { timeoutMs: 180_000 }));
await vi.advanceTimersByTimeAsync(0);
const aborted = new AbortController();
const cancelledOperation = vi.fn(async () => undefined);
const cancelled = admission.withStdinUploadSlot(cancelledOperation, { timeoutMs: 180_000, signal: aborted.signal });
const cancelledAssertion = expect(cancelled).rejects.toThrow("cancelled");
aborted.abort();
await cancelledAssertion;
const expiredOperation = vi.fn(async () => undefined);
const expired = admission.withStdinUploadSlot(expiredOperation, { timeoutMs: 999_999 });
const expiredAssertion = expect(expired).rejects.toThrow("timed out waiting for capacity");
await vi.advanceTimersByTimeAsync(180_000);
await expiredAssertion;
expect(cancelledOperation).not.toHaveBeenCalled();
expect(expiredOperation).not.toHaveBeenCalled();
completions.splice(0).forEach(resolve => resolve());
await Promise.all(occupied);
await expect(admission.withStdinUploadSlot(async () => { throw new Error("failed upload"); }, { timeoutMs: 1000 })).rejects.toThrow("failed upload");
await expect(admission.withStdinUploadSlot(async () => "next", { timeoutMs: 1000 })).resolves.toBe("next");
} finally { vi.useRealTimers(); }
});
it("releases all capacity after failed or cancelled transport commands without replay", async () => {
const admission = createBoatTransportAdmission();
const runner = { execute: vi.fn(async () => { throw new Error("transport cancelled"); }) };
+43 -3
View File
@@ -17,6 +17,8 @@ const apiOrigin = "https://boat.dev/api/v1";
const transportCache = new Map<string, Promise<CommandManagedRuntimeRunner>>();
type CommandInput = Parameters<CommandManagedRuntimeRunner["execute"]>[0];
const BOAT_BULK_COMMAND_LIMIT = 4;
/** Reserve SSH admission for lifecycle/identity calls while bulk and legacy commands run. */
export function createBoatTransportAdmission() {
let active = 0;
@@ -26,14 +28,14 @@ export function createBoatTransportAdmission() {
function drain() {
while (active < 8) {
const eligible = (entry: (typeof queue)[number]) =>
entry.control || (ordinary < 6 && (!entry.bulk || bulk < 4));
entry.control || (ordinary < 6 && (!entry.bulk || bulk < BOAT_BULK_COMMAND_LIMIT));
let index = queue.findIndex((entry) => entry.control && eligible(entry));
if (index < 0) index = queue.findIndex(eligible);
if (index < 0) break;
queue.splice(index, 1)[0]!.start();
}
}
return async function execute(
async function execute(
runner: CommandManagedRuntimeRunner,
input: CommandInput,
control: boolean,
@@ -72,7 +74,44 @@ export function createBoatTransportAdmission() {
queue.push(entry);
drain();
});
}
// All runner views for a machine share these slots. Wait outside execute so
// one upload cannot consume another upload's SSH command deadline in queue.
let uploads = 0;
const uploadQueue: Array<{ start: () => void }> = [];
function drainUploads() {
while (uploads < BOAT_BULK_COMMAND_LIMIT && uploadQueue.length) uploadQueue.shift()!.start();
}
const withStdinUploadSlot: NonNullable<CommandManagedRuntimeRunner["withStdinUploadSlot"]> = (operation, options) => {
if (uploadQueue.length >= 128) return Promise.reject(new ComputerError("conflict", "Computer upload admission is busy"));
return new Promise((resolve, reject) => {
const cleanup = () => { clearTimeout(timer); options.signal?.removeEventListener("abort", aborted); };
const remove = (message: string) => {
const index = uploadQueue.indexOf(entry);
if (index < 0) return;
uploadQueue.splice(index, 1);
cleanup();
reject(new ComputerError("provider_error", message));
};
const aborted = () => remove("Computer upload admission was cancelled");
const entry = { start: () => {
cleanup();
uploads++;
void Promise.resolve().then(operation).then(resolve, reject).finally(() => {
uploads--;
drainUploads();
});
} };
const timer = setTimeout(() => remove("Computer upload timed out waiting for capacity"),
Math.max(1, Math.min(options.timeoutMs, 180_000)));
uploadQueue.push(entry);
options.signal?.addEventListener("abort", aborted, { once: true });
if (options.signal?.aborted) aborted();
else drainUploads();
});
};
return Object.assign(execute, { withStdinUploadSlot });
}
const transportAdmission = new Map<string, ReturnType<typeof createBoatTransportAdmission>>();
const desktopCache = new Map<
@@ -358,7 +397,8 @@ export function boatBackend(
args: ["--signal=KILL", `${input.timeoutMs! / 1000}s`, input.command, ...(input.args ?? [])],
}),
} : raw;
return { ...raw, execute: (input) => execute(bounded, input, options?.control === true) };
return { ...raw, withStdinUploadSlot: admission.withStdinUploadSlot,
execute: (input) => execute(bounded, input, options?.control === true) };
}
async function fileTransport(
record: ComputerRecord,
@@ -1,3 +1,4 @@
import { createBoatTransportAdmission } from "../adapters/boat.js";
import { describe, it, expect, vi } from "vitest";
import {
existsSync,
@@ -115,6 +116,28 @@ function fixture() {
};
}
describe("computer ownership", () => {
it("rechecks owner authority after a shared upload slot wait before SSH dispatch", async () => {
const f = fixture();
const admission = createBoatTransportAdmission();
const execute = vi.fn();
vi.mocked(f.backend.runner).mockResolvedValue({ execute, withStdinUploadSlot: admission.withStdinUploadSlot });
await f.attach();
const binding = await f.admit();
expect(binding.runner.withStdinUploadSlot).toBe(binding.process.runner.withStdinUploadSlot);
const completions: Array<() => void> = [];
const occupied = Array.from({ length: 4 }, () => admission.withStdinUploadSlot(
() => new Promise<void>(resolve => completions.push(resolve)), { timeoutMs: 180_000 }));
await Promise.resolve();
const queued = binding.process.runner.withStdinUploadSlot!(
() => binding.process.runner.execute({ command: "must-not-launch", stdin: "payload", timeoutMs: 180_000 }),
{ timeoutMs: 180_000 });
const assertion = expect(queued).rejects.toThrow(/expired|retired/);
await f.service.retire({ ...f.scope, owner: binding.owner });
completions.forEach(resolve => resolve());
await Promise.all(occupied);
await assertion;
expect(execute).not.toHaveBeenCalled();
});
it("allocates outside guest ephemeral and occupied ports while reserving live owners", async () => {
const f = fixture();
await f.attach();
@@ -1,6 +1,7 @@
import { randomUUID } from "node:crypto";
import type { ComputerRepository, ComputerBackend } from "./ports.js";
import {
COMPUTER_RETIREMENT_GRACE_MS as RETIREMENT_GRACE_MS,
ComputerError,
ComputerStopPendingError,
assertAdmission,
@@ -25,8 +26,6 @@ class ComputerSessionBusyError extends ComputerError {
}
}
// This is shutdown time only; admission and ordinary work stop at the warm deadline.
const RETIREMENT_GRACE_MS = 30_000;
const FILE_OPERATION_BUDGET_MS = 120_000;
const FILE_OPERATION_SETTLEMENT_MS = 5000;
type Scope = { companyId: string; environmentId: string };
@@ -228,6 +227,7 @@ export function createComputerService(
const runnerFor = (
processScoped: boolean,
): CommandManagedRuntimeRunner => ({
withStdinUploadSlot: (processScoped ? control : raw).withStdinUploadSlot,
execute: async (input) => {
const current = await (processScoped
? processScope()
@@ -1,3 +1,6 @@
/** Shutdown only; ordinary work remains fenced at the warm deadline. */
export const COMPUTER_RETIREMENT_GRACE_MS = 30_000;
export type OwnerRef = {
computerId: string;
ownerId: string;
+55 -13
View File
@@ -1,3 +1,4 @@
import { COMPUTER_RETIREMENT_GRACE_MS } from "../../modules/computers/domain/ledger.js";
import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";
import { pendingNativeWorkspaceFinalizationCondition } from "../native-runtime/native-workspace-finalization-state.js";
import { CHAT_COMPLETION_WAKE_REASON } from "../chat-completion-delivery.js";
@@ -30,6 +31,7 @@ import {
import type { Db } from "@paperclipai/db";
import {
agents,
computers,
environmentLeases,
heartbeatRuns,
issues,
@@ -1413,7 +1415,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
const cutoff = new Date(Date.now() - opts.backoffMs);
const rows = await db
.select({ lease: environmentLeases })
.select({ lease: environmentLeases, runAgentId: heartbeatRuns.agentId })
.from(environmentLeases)
.leftJoin(
heartbeatRuns,
@@ -1435,7 +1437,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
.limit(ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE);
let recovered = 0;
for (const { lease } of rows) {
for (const { lease, runAgentId } of rows) {
// A provider resource id names one physical sandbox. A different lease
// row can still hold that same resource in a live status, so this sweep
// must not tear down a sandbox that a different lease still owns.
@@ -1470,19 +1472,59 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
// pending_cleanup sweep can accept the same row in this same tick. A
// fresh timestamp here would push the row inside that sweep's own
// backoff window and delay the teardown by one full tick.
const flipped = await db
const sameLease = and(
eq(environmentLeases.id, lease.id),
eq(environmentLeases.status, "active"),
// A completed heartbeat may retain or replace its bookkeeping while
// this sweep waits for the computer serialization boundary.
eq(environmentLeases.companyId, lease.companyId),
lease.environmentId ? eq(environmentLeases.environmentId, lease.environmentId) : isNull(environmentLeases.environmentId),
lease.heartbeatRunId ? eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId) : isNull(environmentLeases.heartbeatRunId),
lease.metadata === null ? isNull(environmentLeases.metadata) : eq(environmentLeases.metadata, lease.metadata),
lease.provider === null ? isNull(environmentLeases.provider) : eq(environmentLeases.provider, lease.provider),
lease.providerLeaseId === null ? isNull(environmentLeases.providerLeaseId) : eq(environmentLeases.providerLeaseId, lease.providerLeaseId),
);
const claim = (query: Pick<Db, "update">) => query
.update(environmentLeases)
.set({
status: "pending_cleanup",
failureReason: "orphaned_active_lease_recovered",
})
.where(
and(
eq(environmentLeases.id, lease.id),
eq(environmentLeases.status, "active"),
),
)
.set({ status: "pending_cleanup", failureReason: "orphaned_active_lease_recovered" })
.where(sameLease)
.returning({ id: environmentLeases.id });
const binding = parseObject(lease.metadata?.computerOwner);
const flipped = lease.provider === "boat" && lease.metadata?.driver === "computer"
? await db.transaction(async (tx) => {
const [computer] = typeof binding.computerId === "string" && lease.environmentId
? await tx.select().from(computers).where(and(
eq(sql`${computers.id}::text`, binding.computerId),
eq(computers.companyId, lease.companyId),
eq(computers.environmentId, lease.environmentId),
eq(computers.provider, "boat"),
)).for("update") : [];
const owner = Array.isArray(computer?.ledger.owners)
? computer.ledger.owners.map(parseObject).find((value) =>
value.id === binding.ownerId && value.id === lease.providerLeaseId &&
Number.isSafeInteger(binding.generation) && Number(binding.generation) > 0 &&
value.generation === binding.generation && value.kind === "runner" &&
value.runId === lease.heartbeatRunId && runAgentId !== null &&
value.agentId === runAgentId && value.agentId === lease.metadata?.agentId)
: undefined;
const process = parseObject(owner?.process);
const processBound = [process.bootId, process.unitName, process.nonce]
.every((value) => typeof value === "string" && value.length > 0) &&
Number.isSafeInteger(process.launchGeneration) && Number(process.launchGeneration) > 0;
const deadline = typeof owner?.deadline === "string" ? Date.parse(owner.deadline) : NaN;
if (owner?.phase === "warm" && processBound && Number.isFinite(deadline) &&
deadline + COMPUTER_RETIREMENT_GRACE_MS > Date.now()) {
// Native completion retains the durable owner before heartbeat's
// later instruction cleanup can mark the lease retained. A terminal
// run is not permission to kill this bounded warm process. Keep it
// out of the page until a later sweep; expired/mismatched owners
// still follow the ordinary orphan cleanup path.
await tx.update(environmentLeases).set({ updatedAt: new Date() })
.where(sameLease);
return [];
}
return claim(tx);
}) : await claim(db);
if (flipped.length > 0) recovered += 1;
}
@@ -1,3 +1,4 @@
import { createBoatTransportAdmission } from "../../modules/computers/adapters/boat.js";
import { closeWarmNativeSession, compareAndSealRetiredComputerCheckpoint, retiredComputerCheckpointIsSettled, retiredComputerCheckpointUnsettledReason, retireUnlaunchedComputerAttempt, nativeSessionBootstrapTimeoutMs } from "./native-session-executor.js";
import { prepareHeartbeatGitHubLaunchers } from "../heartbeat-github-launchers.js";
import { RemoteProviderPackVerificationError } from "./remote-provider-pack-cache.js";
@@ -2717,6 +2718,62 @@ describe("remote provider checkpoint restores", () => {
} finally { await rm(root, { recursive: true, force: true }); }
});
it("shares Boat upload capacity across concurrent packs before command deadlines begin", async () => {
const root = await mkdtemp(join(tmpdir(), "paperclip-boat-upload-pressure-"));
const sourcePath = join(root, "source");
const bytes = randomBytes(24 * 1024 * 1024);
await writeFile(sourcePath, bytes);
const admission = createBoatTransportAdmission();
const result = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: "" };
let active = 0;
let peak = 0;
let entered!: () => void;
const firstBatch = new Promise<void>(resolve => { entered = resolve; });
let wake: (() => void) | undefined;
const budgets: number[] = [];
const raw = { execute: vi.fn(async (request: { command: string; args?: string[]; stdin?: string; timeoutMs?: number }) => {
if (!request.stdin) {
execFileSync(request.command, request.args ?? []);
return result;
}
active++;
peak = Math.max(peak, active);
if (active === 4) entered();
wake?.(); wake = undefined;
budgets.push(request.timeoutMs!);
try {
await new Promise(resolve => setTimeout(resolve, Math.min(100_000, request.timeoutMs!)));
if (request.timeoutMs! < 100_000) return { ...result, exitCode: null, timedOut: true };
execFileSync(request.command, request.args ?? [], { input: request.stdin });
return result;
} finally { active--; }
}) };
const runner = {
withStdinUploadSlot: admission.withStdinUploadSlot,
execute: (request: Parameters<typeof admission>[1]) => admission(raw, request, true),
};
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
try {
let settled = false;
const transfer = Promise.allSettled(["one", "two"].map(name => stageRemoteRunnerFile({
target: { kind: "remote", transport: "computer" } as never,
runner: runner as never, sourcePath, targetPath: join(root, name), mode: 0o700,
}))).then(results => { settled = true; wake?.(); wake = undefined; return results; });
await firstBatch;
await expect(admission(raw, { command: "true", timeoutMs: 1000 }, true)).resolves.toMatchObject({ exitCode: 0 });
while (!settled) {
if (!active) await new Promise<void>(resolve => { wake = resolve; });
if (!settled) await vi.advanceTimersByTimeAsync(100_000);
}
const results = await transfer;
expect(results.map(result => result.status)).toEqual(["fulfilled", "fulfilled"]);
expect(peak).toBe(4);
expect(budgets).toHaveLength(12);
expect(budgets.every(budget => budget > 99_999)).toBe(true);
for (const name of ["one", "two"]) expect((await readFile(join(root, name))).equals(bytes)).toBe(true);
} finally { vi.useRealTimers(); await rm(root, { recursive: true, force: true }); }
});
it("shares eight upload slots across concurrent artifact transfers", async () => {
const root = await mkdtemp(join(tmpdir(), "paperclip-runner-upload-slots-"));
const sourcePath = join(root, "source");
@@ -10545,7 +10545,7 @@ export async function stageRemoteRunnerFile(input: {
}
const remoteUpload = `${input.targetPath}.upload-${randomUUID()}`;
try {
await uploadRemoteRunnerFileChunks({ sourcePath: input.sourcePath, remotePath: remoteUpload, run });
await uploadRemoteRunnerFileChunks({ sourcePath: input.sourcePath, remotePath: remoteUpload, run, runner });
await run(`chmod ${input.mode.toString(8)} ${quote(remoteUpload)} && mv -f ${quote(remoteUpload)} ${quote(input.targetPath)}`);
} finally {
await run(`rm -f ${quote(remoteUpload)}`).catch(() => undefined);
@@ -10572,6 +10572,7 @@ async function withRemoteArtifactUploadSlot(operation: () => Promise<void>): Pro
}
async function uploadRemoteRunnerFileChunks(input: {
runner: CommandManagedRuntimeRunner;
sourcePath: string;
remotePath: string;
run: (script: string, stdin?: string) => Promise<void>;
@@ -10597,9 +10598,12 @@ async function uploadRemoteRunnerFileChunks(input: {
// Disjoint fixed offsets permit bounded parallel upload and idempotent
// writes. The final digest rejects missing, duplicated, or partial bytes.
const offset = chunkIndex++;
pending.push(withRemoteArtifactUploadSlot(() => run(
const upload = () => run(
`base64 -d | dd of=${quote(remotePath)} bs=4194304 seek=${offset} conv=notrunc 2>/dev/null`,
chunk.toString("base64"))).catch((error) => { uploadError ??= error; }));
chunk.toString("base64"));
pending.push(withRemoteArtifactUploadSlot(() => input.runner.withStdinUploadSlot
? input.runner.withStdinUploadSlot(upload, { timeoutMs: 180_000 })
: upload()).catch((error) => { uploadError ??= error; }));
if (pending.length === REMOTE_ARTIFACT_UPLOAD_CONCURRENCY) {
await Promise.all(pending);
pending.length = 0;
@@ -10708,7 +10712,7 @@ export async function stageRemoteRunnerDirectory(input: {
(error) => error ? reject(error) : resolve());
});
await uploadRemoteRunnerFileChunks({
sourcePath: archivePath, remotePath: remoteArchive, run, onProgress: input.onProgress,
sourcePath: archivePath, remotePath: remoteArchive, run, runner: input.runner, onProgress: input.onProgress,
});
await run(`umask 077 && mkdir -p ${quote(input.targetPath)} && ` +
`tar -xzf ${quote(remoteArchive)} -C ${quote(input.targetPath)} && chmod ${input.mode.toString(8)} ${quote(input.targetPath)}`);
@@ -120,6 +120,19 @@ describe("persistent computer provider pack cache", () => {
expect(() => computerProviderPackCachePath("/agents/../a", digest)).toThrow("identity_invalid");
});
it("preserves transport-shared upload backpressure through the ownership guard", async () => {
const input = await fixture();
const withStdinUploadSlot = vi.fn(async (operation: () => Promise<void>) => operation());
await expect(prepareComputerProviderPackCache({ ...input,
runner: { ...input.runner, withStdinUploadSlot } as CommandManagedRuntimeRunner,
stage: async (root, guarded) => {
expect(guarded.withStdinUploadSlot).toBe(withStdinUploadSlot);
await guarded.withStdinUploadSlot!(() => input.stage(root, guarded), { timeoutMs: 180_000 });
},
})).resolves.toBe("published");
expect(withStdinUploadSlot).toHaveBeenCalledOnce();
});
it("verifies a shared hit before the compression/upload callback and excludes its ownership marker", async () => {
const input = await fixture();
const stage = vi.fn(input.stage); const verify = vi.fn(input.verify);
@@ -298,6 +298,7 @@ export async function prepareComputerProviderPackCache(input: {
}
const stagingRoot = posix.join(posix.dirname(input.cacheRoot), ".uploads", session, posix.basename(input.cacheRoot), identity.attemptId, "pack");
const guardedRunner: CommandManagedRuntimeRunner = {
withStdinUploadSlot: input.runner.withStdinUploadSlot,
execute: (command) => input.runner.execute({ ...command, command: "python3", args: [...args("guard"), command.command, ...(command.args ?? [])] }),
};
try {