mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-10 12:07:09 +02:00
fix(runner-e2e): wait for exact remote runtime before native fixture install
Observe the pinned run process and runtime inode within the original deadline before one observer install. Revalidate identity at installation and baseline, and retain only closed RPC phase/error diagnostics. Preserve action, ownership and cleanup gates. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
31201f4a96
commit
11fdee5f65
3 files changed
+255
-30
No files matched your search
@@ -5,7 +5,7 @@ import { dirname, join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { ObservedStateTimeout, RemoteAdmissionReadError, RunnerApiHttpError, pollUntil } from "./api.js";
|
||||
import { classifyFailure } from "./failure-classifier.js";
|
||||
import { REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS, bindRemoteNativeFixture, type RemoteFixtureApi, type RemoteFixtureDaytona, type RemoteNativeFixture } from "./remote-native-fixtures.js";
|
||||
import { REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS, bindRemoteNativeFixture, remoteNativeFixtureDiagnostics, type RemoteFixtureApi, type RemoteFixtureDaytona, type RemoteNativeFixture } from "./remote-native-fixtures.js";
|
||||
|
||||
type AdmissionEndpoint = "issue" | "run" | "leases";
|
||||
|
||||
@@ -194,7 +194,7 @@ export function createRemoteNativeBootstrap(input: {
|
||||
}
|
||||
try {
|
||||
await input.evidence(`remote-native-bootstrap-startup-${request.runId}.json`, {
|
||||
...lastState, deadlineReached: Date.now() >= input.deadlineAt,
|
||||
...lastState, fixtureDiagnostics: remoteNativeFixtureDiagnostics(error), deadlineReached: Date.now() >= input.deadlineAt,
|
||||
admissionDeadlineReached: Date.now() >= admissionDeadlineAt,
|
||||
});
|
||||
} catch (evidenceError) {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createHash } from "node:crypto";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { mkdtemp, rename, rm, writeFile } from "node:fs/promises";
|
||||
@@ -5,7 +7,9 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Script } from "node:vm";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { bindRemoteNativeFixture, createRemoteTargetWatch, isRemoteRunRoot, parseRemoteProcStat, type RemoteNativeFixtureOptions, type RemoteNativeSnapshot } from "./remote-native-fixtures.js";
|
||||
import { bindRemoteNativeFixture, createRemoteTargetWatch, isRemoteRunRoot, parseRemoteProcStat, remoteNativeFixtureDiagnostics, type RemoteNativeFixtureOptions, type RemoteNativeSnapshot } from "./remote-native-fixtures.js";
|
||||
|
||||
import { createRemoteNativeBootstrap } from "./remote-native-bootstrap.js";
|
||||
|
||||
const hash = (s: string) => `sha256:${createHash("sha256").update(s).digest("hex")}`;
|
||||
const bootId = "12345678-1234-1234-1234-123456789abc";
|
||||
@@ -18,6 +22,7 @@ function snapshot(): RemoteNativeSnapshot {
|
||||
watcher: { complete: true, targetMutationCount: 0, workspaceMutationCount: 0 },
|
||||
processes: { captured: true, root, journal: [root], live: [21] }, scope: { kind: "user_workspace", excludedRuntime: { relativePath: ".paperclip-runtime/paperclip-runner", absolutePath: "/workspace/.paperclip-runtime/paperclip-runner", dev: "1", ino: "4", runnerExecutableSha256: hash("runnerd") }, observedPrpEnvironmentLeaseId: "workspace-id", prpEnvironmentLeaseIdVerified: false }, setup: { path: "action.txt", sha256: null, published: false }, attached: null };
|
||||
}
|
||||
function readiness() { return { ready: true, binding, root, runtime: { dev: "1", ino: "4", runnerExecutableSha256: hash("runnerd") } }; }
|
||||
function harness() {
|
||||
let lease: Record<string, unknown> = { id: "lease", companyId: "company", environmentId: "environment", heartbeatRunId: "run", provider: "daytona", providerLeaseId: "sandbox", status: "active", releasedAt: null,
|
||||
metadata: { sandboxId: "sandbox", image: authority.image, reuseLease: false, remoteCwd: "/workspace", workspaceSentinel: { path: "/workspace/.paperclip-runtime/reusable-sandbox-lease.json", token: "fixture-sentinel-token", result: "written", runId: "run", providerLeaseId: "sandbox" } } };
|
||||
@@ -33,7 +38,7 @@ function harness() {
|
||||
const request = JSON.parse(Buffer.from(encoded, "base64").toString()); calls.push({ command, request, timeout });
|
||||
if (request.op === "wait") return { exitCode: 0, result: JSON.stringify({ ok: true, result: await terminal }) };
|
||||
if (override) { const result = override(request); if (result !== undefined) return result as { exitCode: number; result: string }; }
|
||||
let result: unknown = structuredClone(current);
|
||||
let result: unknown = request.op === "runtime-ready" ? readiness() : structuredClone(current);
|
||||
if (request.op === "publish") { current.setup = { path: request.path, published: true, sha256: hash(request.text) }; result = current.setup; }
|
||||
if (request.op === "close") result = { closed: true };
|
||||
if (request.op === "arm") result = { armed: true, sealed: false };
|
||||
@@ -72,10 +77,101 @@ describe("remote native lease admission", () => {
|
||||
const h = harness(); h.labels["paperclip-run-id"] = "other";
|
||||
await expect(bindRemoteNativeFixture(h.options)).rejects.toThrow("sandbox_labels"); expect(h.get).toHaveBeenCalledExactlyOnceWith("sandbox"); expect(h.executeCommand).not.toHaveBeenCalled();
|
||||
});
|
||||
it("waits beyond the old 20s install subdeadline for staging, then installs exactly once before action", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(1_000_000);
|
||||
try {
|
||||
const h = harness(); h.options.deadlineAt += 60_000;
|
||||
h.override(r => r.op === "runtime-ready" && Date.now() < 1_021_000 ? { exitCode: 0, result: JSON.stringify({ ok: true, result: { ready: false } }) } : undefined);
|
||||
const pending = bindRemoteNativeFixture(h.options);
|
||||
await vi.advanceTimersByTimeAsync(20_000);
|
||||
expect(h.calls.every(c => c.request.op === "runtime-ready")).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(1000); const fixture = await pending;
|
||||
expect(h.calls.filter(c => c.request.op === "install")).toHaveLength(1);
|
||||
expect(h.calls.find(c => c.request.op === "install")!.timeout).toBe(25);
|
||||
expect(h.calls.at(-1)!.request.op).toBe("arm");
|
||||
expect(h.apiGet).toHaveBeenCalledTimes((h.calls.filter(c => c.request.op === "runtime-ready").length + 2) * 2);
|
||||
await fixture.close();
|
||||
} finally { vi.useRealTimers(); }
|
||||
});
|
||||
it("never installs on a missing runtime and reserves installation and teardown inside the original deadline", async () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(1_000_000);
|
||||
try {
|
||||
const h = harness(); h.override(r => r.op === "runtime-ready" ? { exitCode: 0, result: '{"ok":true,"result":{"ready":false}}' } : undefined);
|
||||
const outcome = bindRemoteNativeFixture(h.options).catch(error => error);
|
||||
await vi.advanceTimersByTimeAsync(18_000);
|
||||
expect((await outcome).message).toContain("readiness_deadline");
|
||||
expect(h.calls.every(c => c.request.op === "runtime-ready")).toBe(true);
|
||||
expect(Date.now()).toBe(h.options.deadlineAt - 42_000);
|
||||
} finally { vi.useRealTimers(); }
|
||||
});
|
||||
it.each(["lease", "binding", "shape", "pin", "unknown-error"])("rejects %s during readiness without installing or publishing", async variant => {
|
||||
const h = harness();
|
||||
h.override(r => {
|
||||
if (r.op !== "runtime-ready") return undefined;
|
||||
if (variant === "lease") { h.setLease({ ...h.lease(), heartbeatRunId: "other" }); return { exitCode: 0, result: '{"ok":true,"result":{"ready":false}}' }; }
|
||||
if (variant === "unknown-error") return { exitCode: 2, result: 'SECRET sdk stderr' };
|
||||
const value: any = readiness();
|
||||
if (variant === "binding") value.binding = { ...binding, leaseId: "foreign" };
|
||||
if (variant === "pin") value.runtime.runnerExecutableSha256 = hash("foreign");
|
||||
if (variant === "shape") value.extra = "secret";
|
||||
return { exitCode: 0, result: JSON.stringify({ ok: true, result: value }) };
|
||||
});
|
||||
const error = await bindRemoteNativeFixture(h.options).catch(error => error);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect(h.calls.every(c => c.request.op === "runtime-ready")).toBe(true);
|
||||
expect(JSON.stringify(remoteNativeFixtureDiagnostics(error))).not.toContain("SECRET");
|
||||
});
|
||||
it.each(["extra", "foreign-phase", "unknown-code", "malformed"])("drops %s remote diagnostic content without retry", async variant => {
|
||||
const h = harness();
|
||||
h.override(r => r.op === "runtime-ready" ? { exitCode: 2, result: variant === "malformed" ? "PRIVATE stderr" : JSON.stringify({ ok: false, diagnostic: { phase: variant === "foreign-phase" ? "close" : r.op, code: variant === "unknown-code" ? "PRIVATE stderr" : "socket_error", ...(variant === "extra" ? { private: "PRIVATE stderr" } : {}) } }) } : undefined);
|
||||
const error = await bindRemoteNativeFixture(h.options).catch(error => error);
|
||||
expect(remoteNativeFixtureDiagnostics(error)).toEqual([{ phase: "runtime-ready", code: "invalid_response" }]);
|
||||
expect(error.message).not.toContain("PRIVATE"); expect(h.calls).toHaveLength(1);
|
||||
});
|
||||
it("persists safe startup diagnostics through the real bootstrap catch without publishing an action", async () => {
|
||||
const h = harness(), evidence = vi.fn(async () => {});
|
||||
h.override(r => r.op === "runtime-ready" ? { exitCode: 2, result: JSON.stringify({ ok: false, diagnostic: { phase: "runtime-ready", code: "runtime_binary_identity" } }) } : undefined);
|
||||
const bootstrap = createRemoteNativeBootstrap({ ...h.options, companyId: "company", environmentId: "environment", agentId: "agent", image: authority.image, evidence,
|
||||
api: { get: (async (path: string) => path === "/api/issues/issue" ? { id: "issue", companyId: "company", assigneeAgentId: "agent" }
|
||||
: path === "/api/heartbeat-runs/run" ? { id: "run", companyId: "company", agentId: "agent", status: "running", executionStage: "preparing" }
|
||||
: [{ ...h.lease(), issueId: "issue" }]) as RemoteNativeFixtureOptions["api"]["get"] },
|
||||
}, async options => bindRemoteNativeFixture({ ...h.options, actionFile: options.actionFile }));
|
||||
bootstrap.prompt("nonce");
|
||||
const actionPrompt = vi.fn(async () => "PRIVATE ACTION");
|
||||
await expect(bootstrap.bindAndRelease({ issueId: "issue", runId: "run", targets: ["result.txt"], actionPrompt })).rejects.toThrow("runtime_binary_identity");
|
||||
expect(actionPrompt).not.toHaveBeenCalled();
|
||||
expect(evidence).toHaveBeenCalledWith("remote-native-bootstrap-startup-run.json", expect.objectContaining({
|
||||
phase: "observer_setup", fixtureDiagnostics: [{ phase: "runtime-ready", code: "runtime_binary_identity" }],
|
||||
}));
|
||||
expect(JSON.stringify(evidence.mock.calls)).not.toContain("PRIVATE ACTION");
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready"]);
|
||||
});
|
||||
it.each(["pid", "ppid", "startTicks", "bootId"])("rejects changed %s between readiness and baseline despite extra observer process fields", async key => {
|
||||
const h = harness();
|
||||
const changed = { state: "S", group: 21, ...root, [key]: key === "bootId" ? "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" : key === "startTicks" ? "999" : 99 };
|
||||
h.current.processes = { captured: true, root: changed, journal: [changed], live: [changed.pid] };
|
||||
await expect(bindRemoteNativeFixture(h.options)).rejects.toThrow("runtime_identity_changed");
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "close"]);
|
||||
});
|
||||
it("retains the closed read phase on transport errors without leaking text", async () => {
|
||||
const h = harness(), f = await bindRemoteNativeFixture(h.options);
|
||||
h.override(r => r.op === "read" ? { exitCode: 2, result: JSON.stringify({ ok: false, diagnostic: { phase: "read", code: "socket_error" } }) } : undefined);
|
||||
const error = await f.readFile("result.txt").catch(error => error);
|
||||
expect(remoteNativeFixtureDiagnostics(error)).toEqual([{ phase: "read", code: "socket_error" }]);
|
||||
await f.close();
|
||||
});
|
||||
it("retains only allowlisted install and cleanup diagnostics with no uncertain retry", async () => {
|
||||
const h = harness();
|
||||
h.override(r => ["install", "close"].includes(r.op) ? { exitCode: 2, result: JSON.stringify({ ok: false, diagnostic: { phase: r.op, code: r.op === "install" ? "runtime_identity_changed" : "socket_error" } }) } : undefined);
|
||||
const error = await bindRemoteNativeFixture(h.options).catch(error => error);
|
||||
expect(error.message).toContain("startup_failed_cleanup_unproven");
|
||||
expect(remoteNativeFixtureDiagnostics(error)).toEqual([{ phase: "close", code: "socket_error" }, { phase: "install", code: "runtime_identity_changed" }]);
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "close"]);
|
||||
});
|
||||
it("arms before publish, binds long receipt before teardown and preserves exact final bytes", async () => {
|
||||
const h = harness(), f = await bindRemoteNativeFixture(h.options);
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["install", "wait", "arm"]); expect(f.baseline.processes.live).toEqual([21]);
|
||||
expect(h.calls[1]!.timeout).toBeLessThanOrEqual(45); expect(h.calls[1]!.request.timeoutMs).toBe(h.calls[1]!.timeout! * 1000);
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "wait", "arm"]); expect(f.baseline.processes.live).toEqual([21]);
|
||||
expect(h.calls[2]!.timeout).toBeLessThanOrEqual(45); expect(h.calls[2]!.request.timeoutMs).toBe(h.calls[2]!.timeout! * 1000);
|
||||
await f.publishAction("action.txt", "write only result.txt");
|
||||
await expect(f.publishAction("action.txt", "retry")).rejects.toThrow("publish_bound");
|
||||
const bytes = "\nUnicode 🪴 literal \\n\n";
|
||||
@@ -83,7 +179,7 @@ describe("remote native lease admission", () => {
|
||||
final.targets["result.txt"] = { ...final.targets["result.txt"]!, absent: false, sha256: hash(bytes), mutationCount: 1 };
|
||||
h.resolveTerminal(final); h.apiGet.mockRejectedValue(new Error("lease already deleted"));
|
||||
expect((await f.finish()).processes.live).toEqual([]); expect((await f.readFile("result.txt")).toString()).toBe(bytes);
|
||||
await f.close(); expect(h.calls.map(c => c.request.op)).toEqual(["install", "wait", "arm", "publish"]);
|
||||
await f.close(); expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "wait", "arm", "publish"]);
|
||||
});
|
||||
it("fails closed when lease deletion beats the terminal receipt", async () => {
|
||||
const h = harness(), f = await bindRemoteNativeFixture(h.options); await f.publishAction("action.txt", "task");
|
||||
@@ -115,12 +211,12 @@ describe("remote native lease admission", () => {
|
||||
it("cleans only its admitted observer on a failed startup receipt and never publishes", async () => {
|
||||
const h = harness(); h.current.processes = { captured: false, root: null, journal: [], live: [] };
|
||||
await expect(bindRemoteNativeFixture(h.options)).rejects.toThrow("bootstrap_not_held");
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["install", "close"]);
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "close"]);
|
||||
});
|
||||
it("refuses action publication without a confirmed receipt channel", async () => {
|
||||
const h = harness(); h.override(r => r.op === "arm" ? { exitCode: 0, result: JSON.stringify({ ok: true, result: { armed: false, sealed: false } }) } : undefined);
|
||||
await expect(bindRemoteNativeFixture(h.options)).rejects.toThrow("receipt_channel_not_armed");
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["install", "wait", "arm", "close"]);
|
||||
expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "wait", "arm", "close"]);
|
||||
});
|
||||
it("bounds malformed command output and does not replay an uncertain publish", async () => {
|
||||
const h = harness(), f = await bindRemoteNativeFixture(h.options);
|
||||
@@ -131,12 +227,12 @@ describe("remote native lease admission", () => {
|
||||
});
|
||||
it("ships syntactically valid closed Node programs with exact binary and no provider env", async () => {
|
||||
const h = harness(); await bindRemoteNativeFixture(h.options);
|
||||
const install = h.calls[0]!; const source = install.request.source as string;
|
||||
const install = h.calls.find(c => c.request.op === "install")!; const source = install.request.source as string;
|
||||
expect(() => new Script(source)).not.toThrow(); expect(source).not.toContain("__name(");
|
||||
const rpcQuoted = install.command.match(/ -e (.+) '[A-Za-z0-9+/=]+'$/su)![1]!;
|
||||
const rpc = rpcQuoted.slice(1, -1).replaceAll("'\\''", "'");
|
||||
expect(() => new Script(rpc)).not.toThrow();
|
||||
expect(rpc).toContain("startedAt+20000"); expect(install.timeout).toBeLessThanOrEqual(27); expect(rpc).toContain("r.timeoutMs-(Date.now()-startedAt)");
|
||||
expect(rpc).not.toContain("startedAt+20000"); expect(install.timeout).toBeLessThanOrEqual(27); expect(rpc).toContain("r.timeoutMs-(Date.now()-startedAt)");
|
||||
expect(source).toContain("/proc/"); expect(source).toContain("workspaceWatch"); expect(source).toContain("finalReceipt.files");
|
||||
expect(install.command).toMatch(/^\/usr\/bin\/env -i PATH=\/usr\/bin:\/bin /u);
|
||||
expect(install.request.config.runnerdSha256).toBe(hash("runnerd"));
|
||||
@@ -220,9 +316,14 @@ describe("independent filesystem and process observations", () => {
|
||||
});
|
||||
|
||||
describe("actual generated observer state machine", () => {
|
||||
async function observerHarness(deferStartup = false) {
|
||||
async function observerHarness(deferStartup = false, transformed?: { command: string; observer: string }) {
|
||||
const h = harness(); await bindRemoteNativeFixture(h.options);
|
||||
const { source, config } = h.calls[0]!.request;
|
||||
const install = h.calls.find(c => c.request.op === "install")!;
|
||||
if (transformed) {
|
||||
install.request.source = transformed.observer;
|
||||
install.command = transformed.command;
|
||||
}
|
||||
const { source, config } = install.request;
|
||||
const intervals: Array<() => void> = [], timers: Array<{ fn: () => void; ms: number }> = [];
|
||||
const proc = new Map<number, { ppid: number; group: number; ticks: string; argv: string[] }>([[21, { ppid: 1, group: 21, ticks: "100", argv: ["/workspace/.paperclip-runtime/paperclip-runner/bin/paperclip-runnerd", "--run-id", "run", "--environment-lease-id", "workspace-id", "--lifecycle-mode", "per_turn", "--state-dir", "/workspace/.paperclip-runtime/paperclip-runner/sessions/" + "a".repeat(64) + "/runner"] }]]);
|
||||
const files = new Map<string, Buffer>([[`${config.root}/observer.cjs`, Buffer.from(source)], [config.sentinel.path, Buffer.from(JSON.stringify({ version: 1, provider: "daytona", token: config.sentinel.token, companyId: "company", environmentId: "environment" }))]]);
|
||||
@@ -297,7 +398,7 @@ describe("actual generated observer state machine", () => {
|
||||
const replies: any[] = [], socket = Object.assign(new EventEmitter(), { end: (value: string) => replies.push(JSON.parse(value)), destroy: vi.fn() });
|
||||
handlers[0]!(socket); socket.emit("data", Buffer.from(JSON.stringify({ op, nonce: config.nonce, ...args }) + "\n")); return replies;
|
||||
}
|
||||
return { request, proc, files, watches, fs, intervals, timers, config, handlers, children, symbolicLinks, context, server, directories, listeners, install: h.calls[0]!, replaceRuntimeRoot() { runtimeInode = 999n; } };
|
||||
return { request, proc, files, watches, fs, intervals, timers, config, handlers, children, symbolicLinks, context, server, directories, listeners, install: h.calls.find(c => c.request.op === "install")!, replaceRuntimeRoot() { runtimeInode = 999n; } };
|
||||
}
|
||||
async function generatedRpc(o: Awaited<ReturnType<typeof observerHarness>>, request: Record<string, unknown>, mutateSource = (source: string) => source) {
|
||||
const quoted = o.install.command.match(/ -e (.+) '[A-Za-z0-9+/=]+'$/su)![1]!;
|
||||
@@ -341,6 +442,72 @@ describe("actual generated observer state machine", () => {
|
||||
await new Promise<void>(resolve => setImmediate(resolve));
|
||||
return { output, process, forwarded, spawn, connections };
|
||||
}
|
||||
it("executes readiness and installation after the actual pinned tsx production transform", async () => {
|
||||
const modulePath = fileURLToPath(new URL("./remote-native-fixtures.ts", import.meta.url));
|
||||
const loaderPath = fileURLToPath(new URL("../../cli/node_modules/tsx/dist/loader.mjs", import.meta.url));
|
||||
const script = `
|
||||
import { bindRemoteNativeFixture } from ${JSON.stringify(modulePath)};
|
||||
const binding = ${JSON.stringify(binding)}, ready = ${JSON.stringify(readiness())};
|
||||
const lease = { id: binding.leaseId, companyId: binding.companyId, environmentId: binding.environmentId, heartbeatRunId: binding.runId,
|
||||
provider: 'daytona', providerLeaseId: binding.sandboxId, status: 'active', releasedAt: null,
|
||||
metadata: { sandboxId: binding.sandboxId, image: binding.image, reuseLease: false, remoteCwd: binding.remoteCwd,
|
||||
workspaceSentinel: { path: binding.remoteCwd+'/.paperclip-runtime/reusable-sandbox-lease.json', token: 'fixture-sentinel-token', result: 'written', runId: binding.runId, providerLeaseId: binding.sandboxId } } };
|
||||
let programs;
|
||||
const sandbox = { id: binding.sandboxId, labels: { 'paperclip-provider':'daytona','paperclip-company-id':binding.companyId,'paperclip-environment-id':binding.environmentId,'paperclip-run-id':binding.runId,'paperclip-reuse-lease':'false' },
|
||||
process: { async executeCommand(command) {
|
||||
const encoded = command.slice(command.lastIndexOf(" '")+2,-1), request = JSON.parse(Buffer.from(encoded,'base64'));
|
||||
if(request.op==='runtime-ready') return {exitCode:0,result:JSON.stringify({ok:true,result:ready})};
|
||||
if(request.op==='install') programs = { command, observer:request.source };
|
||||
return {exitCode:0,result:JSON.stringify({ok:true,result:request.op==='close'?{closed:true}:{}})};
|
||||
} } };
|
||||
try { await bindRemoteNativeFixture({ authority: ${JSON.stringify(authority)}, api:{get:async p=>p.includes('/environments/')?[lease]:lease}, daytona:{get:async()=>sandbox},
|
||||
sdkVersion:'0.203.0',nodeSha256:${JSON.stringify(hash("node"))},runnerdSha256:${JSON.stringify(hash("runnerd"))},targets:['result.txt'],actionFile:'action.txt',deadlineAt:Date.now()+60000 }); } catch {}
|
||||
if(!programs) throw Error('No captured generated programs');
|
||||
process.stdout.write(JSON.stringify(programs));
|
||||
`;
|
||||
const transformed = JSON.parse(execFileSync(process.execPath, ["--import", loaderPath, "--input-type=module", "-e", script], {
|
||||
env: { PATH: "/usr/bin:/bin", HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, TSX_DISABLE_CACHE: "1" }, timeout: 10_000, maxBuffer: 256 * 1024, encoding: "utf8",
|
||||
}));
|
||||
expect(transformed.command).not.toContain("__name("); expect(transformed.observer).not.toContain("__name(");
|
||||
const o = await observerHarness(true, transformed);
|
||||
const ready = await generatedRpc(o, { ...o.install.request, op: "runtime-ready" });
|
||||
expect(JSON.parse(ready.output)).toEqual({ ok: true, result: readiness() });
|
||||
const installed = await generatedRpc(o, o.install.request);
|
||||
expect(installed.process.exitCode).toBe(0); expect(JSON.parse(installed.output).result.complete).toBe(true);
|
||||
const closed = await generatedRpc(o, { op: "close", root: o.config.root, nonce: o.config.nonce, nodeSha256: hash("node"), timeoutMs: 10_000 });
|
||||
expect(JSON.parse(closed.output)).toEqual({ ok: true, result: { closed: true } });
|
||||
o.timers.find(t => t.ms === 100)!.fn(); expect(o.watches.every(w => w.closed)).toBe(true);
|
||||
});
|
||||
it("executes the actual readiness probe without creating an observer, then revalidates its exact identity at install", async () => {
|
||||
const o = await observerHarness(true);
|
||||
const ready = await generatedRpc(o, { ...o.install.request, op: "runtime-ready" });
|
||||
expect(JSON.parse(ready.output)).toEqual({ ok: true, result: readiness() });
|
||||
expect(ready.spawn).not.toHaveBeenCalled(); expect(o.fs.mkdirSync).not.toHaveBeenCalled(); expect(o.watches).toHaveLength(0);
|
||||
o.replaceRuntimeRoot();
|
||||
const changed = await generatedRpc(o, o.install.request);
|
||||
expect(JSON.parse(changed.output).diagnostic.code).toBe("runtime_identity_changed");
|
||||
expect(changed.spawn).not.toHaveBeenCalled(); expect(o.fs.mkdirSync).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each(["absent", "foreign-run", "foreign-group", "ambiguous", "wrong-binary", "symlink", "disappeared"])("checks generated runtime readiness: %s", async variant => {
|
||||
const o = await observerHarness(true), runtimePath = "/workspace/.paperclip-runtime/paperclip-runner";
|
||||
if (variant === "absent") o.directories.delete(runtimePath);
|
||||
if (variant === "foreign-run") o.proc.get(21)!.argv[2] = "foreign";
|
||||
if (variant === "foreign-group") o.proc.get(21)!.group = 99;
|
||||
if (variant === "ambiguous") o.proc.set(22, { ...o.proc.get(21)!, group: 22 });
|
||||
if (variant === "symlink") o.symbolicLinks.add(runtimePath);
|
||||
if (variant === "disappeared") o.proc.clear();
|
||||
const request = { ...o.install.request, op: variant === "disappeared" ? "install" : "runtime-ready", config: { ...o.config, ...(variant === "wrong-binary" ? { runnerdSha256: hash("wrong") } : {}) } };
|
||||
const result = await generatedRpc(o, request), parsed = JSON.parse(result.output);
|
||||
if (["absent", "foreign-run", "foreign-group"].includes(variant)) expect(parsed).toEqual({ ok: true, result: { ready: false } });
|
||||
else expect(parsed).toEqual({ ok: false, diagnostic: { phase: request.op, code: { ambiguous: "ambiguous_run_root", "wrong-binary": "runtime_binary_identity", symlink: "runtime_root_identity", disappeared: "runtime_not_ready" }[variant] } });
|
||||
expect(result.spawn).not.toHaveBeenCalled(); expect(o.fs.mkdirSync).not.toHaveBeenCalled();
|
||||
});
|
||||
it("uses the closed read phase for actual generated socket errors", async () => {
|
||||
const o = await observerHarness(true);
|
||||
const read = await generatedRpc(o, { op: "read", root: o.config.root, nonce: o.config.nonce, nodeSha256: hash("node"), timeoutMs: 10_000, path: "result.txt" });
|
||||
expect(JSON.parse(read.output)).toEqual({ ok: false, diagnostic: { phase: "read", code: "socket_error" } });
|
||||
expect(read.spawn).not.toHaveBeenCalled();
|
||||
});
|
||||
it("executes generated install through the bounded observer socket, then closes the exact observer", async () => {
|
||||
const o = await observerHarness(true);
|
||||
expect(Buffer.byteLength(o.install.request.source)).toBeGreaterThan(8192);
|
||||
@@ -348,7 +515,13 @@ describe("actual generated observer state machine", () => {
|
||||
expect(installed.process.exitCode).toBe(0);
|
||||
const baseline = JSON.parse(installed.output);
|
||||
expect(baseline.ok).toBe(true); expect(baseline.result.complete).toBe(true);
|
||||
expect(baseline.result.processes.root.pid).toBe(21);
|
||||
expect(baseline.result.processes.root).toMatchObject({ ...root, group: 21, state: "S" });
|
||||
// Feed the actual generated observer receipt through the real host reader,
|
||||
// including parseRemoteProcStat's extra fields and different key order.
|
||||
const h = harness(); h.override(request => request.op === "install" ? { exitCode: 0, result: installed.output } : undefined);
|
||||
const fixture = await bindRemoteNativeFixture(h.options);
|
||||
expect(fixture.baseline.processes.root).toEqual(baseline.result.processes.root);
|
||||
await fixture.close();
|
||||
expect(installed.spawn).toHaveBeenCalledTimes(1);
|
||||
expect(installed.forwarded.map(value => JSON.parse(value))).toEqual([{ op: "snapshot", nonce: o.config.nonce }]);
|
||||
expect(Buffer.byteLength(installed.forwarded[0]!)).toBeLessThan(8192);
|
||||
@@ -369,7 +542,7 @@ describe("actual generated observer state machine", () => {
|
||||
const mutant = source.replace("fs.mkdirSync(c.root,{mode:0o700});", "");
|
||||
expect(mutant).not.toBe(source); return mutant;
|
||||
});
|
||||
expect(installed.process.exitCode).toBe(2); expect(installed.output).toBe("");
|
||||
expect(installed.process.exitCode).toBe(2); expect(JSON.parse(installed.output).diagnostic).toEqual({ phase: "install", code: "remote_unknown" });
|
||||
expect(installed.spawn).not.toHaveBeenCalled(); expect(installed.connections).toHaveLength(0);
|
||||
expect(o.files.has(`${o.config.root}/observer.cjs`)).toBe(false);
|
||||
expect(o.directories.has(o.config.root)).toBe(false); expect(o.watches).toHaveLength(0);
|
||||
@@ -380,7 +553,7 @@ describe("actual generated observer state machine", () => {
|
||||
const mutant = source.replace("server.listen(path.join(config.root,'control.sock'));", "");
|
||||
expect(mutant).not.toBe(source);
|
||||
const installed = await generatedRpc(o, { ...o.install.request, source: mutant });
|
||||
expect(installed.process.exitCode).toBe(2); expect(installed.output).toBe("");
|
||||
expect(installed.process.exitCode).toBe(2); expect(JSON.parse(installed.output).diagnostic).toEqual({ phase: "install", code: "socket_error" });
|
||||
expect(installed.spawn).toHaveBeenCalledTimes(1); expect(o.handlers).toHaveLength(1);
|
||||
expect(o.server.listen).not.toHaveBeenCalled(); expect(o.listeners.size).toBe(0);
|
||||
expect(installed.connections).toHaveLength(0); expect(installed.forwarded).toHaveLength(0);
|
||||
@@ -393,7 +566,7 @@ describe("actual generated observer state machine", () => {
|
||||
it("still rejects an oversized control request and can close its observer afterward", async () => {
|
||||
const o = await observerHarness();
|
||||
const oversized = await generatedRpc(o, { op: "snapshot", root: o.config.root, nonce: o.config.nonce, nodeSha256: hash("node"), timeoutMs: 10_000, unexpected: "x".repeat(8192) });
|
||||
expect(oversized.process.exitCode).toBe(2); expect(oversized.output).toBe("");
|
||||
expect(oversized.process.exitCode).toBe(2); expect(JSON.parse(oversized.output).diagnostic).toEqual({ phase: "snapshot", code: "socket_error" });
|
||||
expect(oversized.connections[0]!.observer.destroy).toHaveBeenCalled();
|
||||
const closed = await generatedRpc(o, { op: "close", root: o.config.root, nonce: o.config.nonce, nodeSha256: hash("node"), timeoutMs: 10_000 });
|
||||
expect(JSON.parse(closed.output)).toEqual({ ok: true, result: { closed: true } });
|
||||
|
||||
@@ -22,6 +22,23 @@ function relative(value: string): string {
|
||||
fail(value.length <= 240 && /^[a-zA-Z0-9._/-]+$/u.test(value) && !value.startsWith("/") && value.split("/").every(part => part !== "" && part !== "." && part !== ".."), "unsafe_relative_target");
|
||||
return value;
|
||||
}
|
||||
// Only closed diagnostic enums cross the remote boundary; SDK errors and output
|
||||
// are never retained. These diagnostics explain failed evidence, not qualification.
|
||||
const RPC_PHASES = ["runtime-ready", "install", "wait", "close", "arm", "publish", "snapshot", "attached", "read"] as const;
|
||||
const RPC_CODES = ["node_identity", "sentinel_type", "sentinel", "cwd", "runtime_root_identity", "runtime_binary_identity", "proc_bound", "ambiguous_run_root", "runtime_not_ready", "runtime_identity_changed", "invalid_proc_identity", "invalid_proc_fields", "socket_error", "socket_timeout", "output_bound", "rpc_deadline", "remote_unknown", "transport_failure", "invalid_response", "readiness_deadline"] as const;
|
||||
type RpcPhase = typeof RPC_PHASES[number];
|
||||
type RpcCode = typeof RPC_CODES[number];
|
||||
interface RpcDiagnostic { phase: RpcPhase; code: RpcCode }
|
||||
class RemoteFixtureError extends Error {
|
||||
constructor(message: string, readonly diagnostic: RpcDiagnostic, options?: ErrorOptions) { super(message, options); }
|
||||
}
|
||||
export function remoteNativeFixtureDiagnostics(error: unknown): RpcDiagnostic[] {
|
||||
const result: RpcDiagnostic[] = [];
|
||||
for (let current = error, depth = 0; current instanceof Error && depth < 3; current = current.cause, depth++) {
|
||||
if (current instanceof RemoteFixtureError) result.push({ ...current.diagnostic });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
export interface RemoteNativeAuthority { companyId: string; environmentId: string; runId: string; leaseId: string; sandboxId: string; image: string }
|
||||
export interface RemoteNativeBinding extends RemoteNativeAuthority { remoteCwd: string }
|
||||
export interface RemoteProcessIdentity { pid: number; ppid: number; startTicks: string; bootId: string }
|
||||
@@ -167,15 +184,18 @@ function observerSource() {
|
||||
.replace("WATCH_TARGET", () => createRemoteTargetWatch.toString()).replace("ATTACHED_CLIENT", () => JSON.stringify(ATTACHED_CLIENT));
|
||||
}
|
||||
const RPC = String.raw`const fs=require('node:fs'),net=require('node:net'),cp=require('node:child_process'),crypto=require('node:crypto');const r=JSON.parse(Buffer.from(process.argv[1],'base64').toString());const hash=x=>'sha256:'+crypto.createHash('sha256').update(x).digest('hex');
|
||||
const startedAt=Date.now();if(!Number.isInteger(r.timeoutMs)||r.timeoutMs<1000||r.timeoutMs>300000)throw Error('rpc_deadline');setTimeout(()=>process.exit(2),r.timeoutMs).unref();
|
||||
const startedAt=Date.now();if(!Number.isInteger(r.timeoutMs)||r.timeoutMs<1000||r.timeoutMs>300000)throw Error('rpc_deadline');setTimeout(()=>{failure('rpc_deadline');process.exit(2)},r.timeoutMs).unref();
|
||||
const parseStat=PARSE_STAT,runRoot=RUN_ROOT;
|
||||
async function waitRuntime(c){const root=c.binding.remoteCwd+'/'+c.runtimeRelative,boot=fs.readFileSync('/proc/sys/kernel/random/boot_id','utf8').trim(),until=Math.min(startedAt+20000,startedAt+r.timeoutMs-1000);while(Date.now()<until){try{const s=fs.lstatSync(root);if(!s.isDirectory()||s.isSymbolicLink()||fs.realpathSync(root)!==root)throw Error('runtime_root_identity');const matches=[];const entries=fs.readdirSync('/proc');if(entries.length>8192)throw Error('proc_bound');for(const name of entries){if(!/^\d+$/.test(name)||Number(name)<2)continue;try{const p=parseStat(Number(name),fs.readFileSync('/proc/'+name+'/stat','utf8'),boot),argv=fs.readFileSync('/proc/'+name+'/cmdline').toString().split('\0').filter(Boolean);if(runRoot(argv,c.binding.runId,p)){if(argv[0]!==root+'/bin/paperclip-runnerd'||hash(fs.readFileSync('/proc/'+name+'/exe'))!==c.runnerdSha256)throw Error('runtime_binary_identity');matches.push(p)}}catch(e){if(e.code!=='ENOENT'&&e.code!=='ESRCH')throw e}}if(matches.length>1)throw Error('ambiguous_run_root');if(matches.length===1)return;}catch(e){if(e.code!=='ENOENT')throw e}await new Promise(resolve=>setTimeout(resolve,50))}throw Error('runtime_not_ready')}
|
||||
(async()=>{let controlRequest=r;if(hash(fs.readFileSync(process.execPath))!==r.nodeSha256)throw Error('node_identity');if(r.op==='install'){const c=r.config;await waitRuntime(c);const st=fs.lstatSync(c.sentinel.path);if(!st.isFile()||st.isSymbolicLink()||st.size>16384||fs.realpathSync(c.sentinel.path)!==c.sentinel.path)throw Error('sentinel_type');const s=JSON.parse(fs.readFileSync(c.sentinel.path,'utf8'));if(s.version!==1||s.provider!=='daytona'||s.token!==c.sentinel.token||s.companyId!==c.binding.companyId||s.environmentId!==c.binding.environmentId)throw Error('sentinel');if(fs.realpathSync(c.binding.remoteCwd)!==c.binding.remoteCwd)throw Error('cwd');fs.mkdirSync(c.root,{mode:0o700});fs.writeFileSync(c.root+'/observer.cjs',r.source,{flag:'wx',mode:0o400});c.observerTtlMs=Math.max(1,c.observerTtlMs-(Date.now()-startedAt));const child=cp.spawn(process.execPath,[c.root+'/observer.cjs',Buffer.from(JSON.stringify(c)).toString('base64')],{detached:true,stdio:'ignore',env:{PATH:'/usr/bin:/bin'}});child.unref();r.root=c.root;controlRequest={op:'snapshot',nonce:c.nonce};}
|
||||
for(let i=0;!fs.existsSync(r.root+'/control.sock')&&i<200;i++)await new Promise(resolve=>setTimeout(resolve,10));const socket=net.connect(r.root+'/control.sock');let output='';socket.setTimeout(Math.max(1,r.timeoutMs-(Date.now()-startedAt)));socket.on('timeout',()=>{socket.destroy();process.exitCode=2});socket.on('error',()=>{process.exitCode=2});socket.on('connect',()=>socket.write(JSON.stringify(controlRequest)+'\n'));socket.on('data',b=>{output+=b;if(Buffer.byteLength(output)>262144){socket.destroy();process.exitCode=2}});socket.on('end',()=>{if(!process.exitCode)process.stdout.write(output)});
|
||||
})().catch(()=>{process.exitCode=2});`;
|
||||
const codes=RPC_CODES;let failed=false;const phase=RPC_PHASES.includes(r.op)?r.op:'install';
|
||||
function failure(code){if(failed)return;failed=true;process.exitCode=2;process.stdout.write(JSON.stringify({ok:false,diagnostic:{phase,code:codes.includes(code)?code:'remote_unknown'}})+'\n')}
|
||||
function admitted(c){const st=fs.lstatSync(c.sentinel.path);if(!st.isFile()||st.isSymbolicLink()||st.size>16384||fs.realpathSync(c.sentinel.path)!==c.sentinel.path)throw Error('sentinel_type');const s=JSON.parse(fs.readFileSync(c.sentinel.path,'utf8'));if(s.version!==1||s.provider!=='daytona'||s.token!==c.sentinel.token||s.companyId!==c.binding.companyId||s.environmentId!==c.binding.environmentId)throw Error('sentinel');if(fs.realpathSync(c.binding.remoteCwd)!==c.binding.remoteCwd)throw Error('cwd')}
|
||||
function runtime(c){const root=c.binding.remoteCwd+'/'+c.runtimeRelative,boot=fs.readFileSync('/proc/sys/kernel/random/boot_id','utf8').trim();let s;try{s=fs.lstatSync(root,{bigint:true})}catch(e){if(e.code==='ENOENT')return {ready:false};throw e}if(!s.isDirectory()||s.isSymbolicLink()||fs.realpathSync(root)!==root)throw Error('runtime_root_identity');const matches=[];const entries=fs.readdirSync('/proc');if(entries.length>8192)throw Error('proc_bound');for(const name of entries){if(!/^\d+$/.test(name)||Number(name)<2)continue;try{const p=parseStat(Number(name),fs.readFileSync('/proc/'+name+'/stat','utf8'),boot),argv=fs.readFileSync('/proc/'+name+'/cmdline').toString().split('\0').filter(Boolean);if(runRoot(argv,c.binding.runId,p)){if(argv[0]!==root+'/bin/paperclip-runnerd'||hash(fs.readFileSync('/proc/'+name+'/exe'))!==c.runnerdSha256)throw Error('runtime_binary_identity');if(p.state!=='Z')matches.push({pid:p.pid,ppid:p.ppid,startTicks:p.startTicks,bootId:p.bootId})}}catch(e){if(e.code!=='ENOENT'&&e.code!=='ESRCH')throw e}}if(matches.length>1)throw Error('ambiguous_run_root');return matches.length===1?{ready:true,binding:c.binding,root:matches[0],runtime:{dev:String(s.dev),ino:String(s.ino),runnerExecutableSha256:c.runnerdSha256}}:{ready:false}}
|
||||
(async()=>{let controlRequest=r;if(hash(fs.readFileSync(process.execPath))!==r.nodeSha256)throw Error('node_identity');if(r.op==='runtime-ready'||r.op==='install'){const c=r.config;admitted(c);const current=runtime(c);if(r.op==='runtime-ready'){process.stdout.write(JSON.stringify({ok:true,result:current})+'\n');return}if(!current.ready)throw Error('runtime_not_ready');if(JSON.stringify(current)!==JSON.stringify(r.expectedRuntime))throw Error('runtime_identity_changed');fs.mkdirSync(c.root,{mode:0o700});fs.writeFileSync(c.root+'/observer.cjs',r.source,{flag:'wx',mode:0o400});c.observerTtlMs=Math.max(1,c.observerTtlMs-(Date.now()-startedAt));const child=cp.spawn(process.execPath,[c.root+'/observer.cjs',Buffer.from(JSON.stringify(c)).toString('base64')],{detached:true,stdio:'ignore',env:{PATH:'/usr/bin:/bin'}});child.unref();r.root=c.root;controlRequest={op:'snapshot',nonce:c.nonce};}
|
||||
for(let i=0;!fs.existsSync(r.root+'/control.sock')&&i<200;i++)await new Promise(resolve=>setTimeout(resolve,10));const socket=net.connect(r.root+'/control.sock');let output='';socket.setTimeout(Math.max(1,r.timeoutMs-(Date.now()-startedAt)));socket.on('timeout',()=>{socket.destroy();failure('socket_timeout')});socket.on('error',()=>{failure('socket_error')});socket.on('connect',()=>socket.write(JSON.stringify(controlRequest)+'\n'));socket.on('data',b=>{output+=b;if(Buffer.byteLength(output)>262144){socket.destroy();failure('output_bound')}});socket.on('end',()=>{if(!process.exitCode)process.stdout.write(output)});
|
||||
})().catch(e=>{failure(typeof e?.message==='string'?e.message:'remote_unknown')});`;
|
||||
|
||||
function rpcSource() {
|
||||
return RPC.replace("PARSE_STAT", () => parseRemoteProcStat.toString()).replace("RUN_ROOT", () => isRemoteRunRoot.toString());
|
||||
return RPC.replace("RPC_CODES", () => JSON.stringify(RPC_CODES)).replace("RPC_PHASES", () => JSON.stringify(RPC_PHASES)).replace("PARSE_STAT", () => parseRemoteProcStat.toString()).replace("RUN_ROOT", () => isRemoteRunRoot.toString());
|
||||
}
|
||||
|
||||
export interface RemoteNativeFixture {
|
||||
@@ -309,8 +329,8 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption
|
||||
]);
|
||||
} finally { if (timer) clearTimeout(timer); }
|
||||
}
|
||||
async function rpc(request: Record<string, unknown>, admitted?: Awaited<ReturnType<typeof admittedSandbox>>) {
|
||||
const available = request.op === "close" ? CLOSE_GRACE_MS : receiptDeadlineAt - Date.now();
|
||||
async function rpc(request: Record<string, unknown>, admitted?: Awaited<ReturnType<typeof admittedSandbox>>, deadlineAt = receiptDeadlineAt) {
|
||||
const available = request.op === "close" ? CLOSE_GRACE_MS : deadlineAt - Date.now();
|
||||
const cap = request.op === "install" ? 27_000 : request.op === "wait" ? 300_000 : 12_000;
|
||||
const budgetMs = Math.floor(Math.min(available, cap) / 1000) * 1000;
|
||||
fail(budgetMs >= 1000, "receipt_deadline");
|
||||
@@ -328,8 +348,17 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption
|
||||
const command = `/usr/bin/env -i PATH=/usr/bin:/bin ${quote(NODE)} -e ${quote(rpcSource())} ${quote(payload)}`;
|
||||
let response: { exitCode: number; result: string };
|
||||
try { response = await sandbox.process.executeCommand(command, binding!.remoteCwd, {}, timeoutMs / 1000); }
|
||||
catch { throw new Error("remote_native_fixture:remote_command_failed_or_deadline"); }
|
||||
fail(response.exitCode === 0 && typeof response.result === "string" && Buffer.byteLength(response.result) <= MAX_OUTPUT, "command_failed_or_output_bound");
|
||||
catch { throw new RemoteFixtureError("remote_native_fixture:remote_command_failed_or_deadline", { phase: request.op as RpcPhase, code: "transport_failure" }); }
|
||||
if (typeof response.result !== "string" || Buffer.byteLength(response.result) > MAX_OUTPUT) throw new RemoteFixtureError("remote_native_fixture:command_failed_or_output_bound", { phase: request.op as RpcPhase, code: "output_bound" });
|
||||
if (response.exitCode !== 0) {
|
||||
let diagnostic: RpcDiagnostic = { phase: request.op as RpcPhase, code: "invalid_response" };
|
||||
try {
|
||||
const value = record(JSON.parse(response.result)), d = record(value.diagnostic);
|
||||
if (value.ok === false && Object.keys(value).sort().join() === "diagnostic,ok" && Object.keys(d).sort().join() === "code,phase"
|
||||
&& d.phase === request.op && RPC_CODES.includes(d.code as RpcCode)) diagnostic = { phase: d.phase as RpcPhase, code: d.code as RpcCode };
|
||||
} catch { /* Never retain remote text. */ }
|
||||
throw new RemoteFixtureError(`remote_native_fixture:command_failed_or_output_bound:${diagnostic.phase}:${diagnostic.code}`, diagnostic);
|
||||
}
|
||||
let parsed: Record<string, unknown>;
|
||||
try { parsed = record(JSON.parse(response.result)); } catch { throw new Error("remote_native_fixture:invalid_observer_json"); }
|
||||
fail(parsed.ok === true, "observer_incomplete");
|
||||
@@ -339,16 +368,39 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption
|
||||
const sandbox = await bounded(admittedSandbox, Math.min(10_000, receiptDeadlineAt - Date.now()));
|
||||
const names = [...targets, ...(options.crossRoot ? ["@cross-root"] : [])];
|
||||
const config = { root, nonce, binding, sentinel, targets, actionFile, crossRoot: options.crossRoot, runtimeRelative: RUNTIME_RELATIVE, runnerdSha256: options.runnerdSha256 };
|
||||
// Lease activation precedes runner artifact staging. Observe the exact pinned
|
||||
// run root before spending the single installation budget. No observer or
|
||||
// action exists during this polling phase, and failures are never retried.
|
||||
const readyDeadline = receiptDeadlineAt - 27_000;
|
||||
let expectedRuntime: Record<string, unknown>;
|
||||
while (true) {
|
||||
if (readyDeadline - Date.now() < 1000) throw new RemoteFixtureError("remote_native_fixture:readiness_deadline", { phase: "runtime-ready", code: "readiness_deadline" });
|
||||
const value = record(await rpc({ op: "runtime-ready", config }, undefined, readyDeadline));
|
||||
if (value.ready === true) {
|
||||
const process = record(value.root), runtime = record(value.runtime);
|
||||
fail(Object.keys(value).sort().join() === "binding,ready,root,runtime" && JSON.stringify(value.binding) === JSON.stringify(binding)
|
||||
&& Object.keys(process).sort().join() === "bootId,pid,ppid,startTicks" && Number.isSafeInteger(process.pid) && Number(process.pid) > 1
|
||||
&& Number.isSafeInteger(process.ppid) && Number(process.ppid) >= 0 && typeof process.startTicks === "string" && /^\d+$/u.test(process.startTicks)
|
||||
&& typeof process.bootId === "string" && /^[a-f0-9-]{36}$/u.test(process.bootId)
|
||||
&& Object.keys(runtime).sort().join() === "dev,ino,runnerExecutableSha256" && typeof runtime.dev === "string" && /^\d+$/u.test(runtime.dev)
|
||||
&& typeof runtime.ino === "string" && /^\d+$/u.test(runtime.ino) && runtime.runnerExecutableSha256 === options.runnerdSha256, "runtime_ready_identity");
|
||||
expectedRuntime = value; break;
|
||||
}
|
||||
fail(value.ready === false && Object.keys(value).length === 1, "runtime_ready_shape");
|
||||
await new Promise(resolve => setTimeout(resolve, Math.min(200, Math.max(0, readyDeadline - Date.now()))));
|
||||
}
|
||||
let baseline: RemoteNativeSnapshot;
|
||||
try {
|
||||
baseline = readSnapshot(await rpc({ op: "install", config, source: observerSource() }, sandbox), binding!, names, actionFile, options.runnerdSha256);
|
||||
baseline = readSnapshot(await rpc({ op: "install", config, expectedRuntime, source: observerSource() }), binding!, names, actionFile, options.runnerdSha256);
|
||||
fail(baseline.complete && baseline.processes.captured && baseline.processes.live.length > 0 && baseline.watcher.complete && !baseline.setup.published, "bootstrap_not_held");
|
||||
fail((["pid", "ppid", "startTicks", "bootId"] as const).every(key => baseline.processes.root?.[key] === record(expectedRuntime.root)[key])
|
||||
&& baseline.scope.excludedRuntime.dev === record(expectedRuntime.runtime).dev && baseline.scope.excludedRuntime.ino === record(expectedRuntime.runtime).ino, "runtime_identity_changed");
|
||||
} catch (error) {
|
||||
// Only the nonce/inode-bound observer can acknowledge this cleanup. If
|
||||
// launch failed before its socket became available, TTL remains a bound,
|
||||
// not a claimed successful cleanup receipt.
|
||||
try { await rpc({ op: "close" }, sandbox); }
|
||||
catch { throw new Error("remote_native_fixture:startup_failed_cleanup_unproven", { cause: error }); }
|
||||
catch (cleanupError) { throw new RemoteFixtureError("remote_native_fixture:startup_failed_cleanup_unproven", remoteNativeFixtureDiagnostics(cleanupError)[0] ?? { phase: "close", code: "remote_unknown" }, { cause: error }); }
|
||||
throw error;
|
||||
}
|
||||
// Start receiving while the lease is still authorized, before publishAction.
|
||||
|
||||
Reference in new issue
Block a user