diff --git a/tests/runner-e2e/remote-native-bootstrap.ts b/tests/runner-e2e/remote-native-bootstrap.ts index af6dfcc593..a9253f71ab 100644 --- a/tests/runner-e2e/remote-native-bootstrap.ts +++ b/tests/runner-e2e/remote-native-bootstrap.ts @@ -5,7 +5,7 @@ import { dirname, join } from "node:path"; import { pathToFileURL } from "node:url"; import { ObservedStateTimeout, RemoteAdmissionReadError, RunnerApiHttpError, pollUntil } from "./api.js"; import { classifyFailure } from "./failure-classifier.js"; -import { REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS, bindRemoteNativeFixture, type RemoteFixtureApi, type RemoteFixtureDaytona, type RemoteNativeFixture } from "./remote-native-fixtures.js"; +import { REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS, bindRemoteNativeFixture, remoteNativeFixtureDiagnostics, type RemoteFixtureApi, type RemoteFixtureDaytona, type RemoteNativeFixture } from "./remote-native-fixtures.js"; type AdmissionEndpoint = "issue" | "run" | "leases"; @@ -194,7 +194,7 @@ export function createRemoteNativeBootstrap(input: { } try { await input.evidence(`remote-native-bootstrap-startup-${request.runId}.json`, { - ...lastState, deadlineReached: Date.now() >= input.deadlineAt, + ...lastState, fixtureDiagnostics: remoteNativeFixtureDiagnostics(error), deadlineReached: Date.now() >= input.deadlineAt, admissionDeadlineReached: Date.now() >= admissionDeadlineAt, }); } catch (evidenceError) { diff --git a/tests/runner-e2e/remote-native-fixtures.test.ts b/tests/runner-e2e/remote-native-fixtures.test.ts index 5c05950bee..5e94e5d958 100644 --- a/tests/runner-e2e/remote-native-fixtures.test.ts +++ b/tests/runner-e2e/remote-native-fixtures.test.ts @@ -1,3 +1,5 @@ +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; import { createHash } from "node:crypto"; import { EventEmitter } from "node:events"; import { mkdtemp, rename, rm, writeFile } from "node:fs/promises"; @@ -5,7 +7,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { Script } from "node:vm"; import { describe, expect, it, vi } from "vitest"; -import { bindRemoteNativeFixture, createRemoteTargetWatch, isRemoteRunRoot, parseRemoteProcStat, type RemoteNativeFixtureOptions, type RemoteNativeSnapshot } from "./remote-native-fixtures.js"; +import { bindRemoteNativeFixture, createRemoteTargetWatch, isRemoteRunRoot, parseRemoteProcStat, remoteNativeFixtureDiagnostics, type RemoteNativeFixtureOptions, type RemoteNativeSnapshot } from "./remote-native-fixtures.js"; + +import { createRemoteNativeBootstrap } from "./remote-native-bootstrap.js"; const hash = (s: string) => `sha256:${createHash("sha256").update(s).digest("hex")}`; const bootId = "12345678-1234-1234-1234-123456789abc"; @@ -18,6 +22,7 @@ function snapshot(): RemoteNativeSnapshot { watcher: { complete: true, targetMutationCount: 0, workspaceMutationCount: 0 }, processes: { captured: true, root, journal: [root], live: [21] }, scope: { kind: "user_workspace", excludedRuntime: { relativePath: ".paperclip-runtime/paperclip-runner", absolutePath: "/workspace/.paperclip-runtime/paperclip-runner", dev: "1", ino: "4", runnerExecutableSha256: hash("runnerd") }, observedPrpEnvironmentLeaseId: "workspace-id", prpEnvironmentLeaseIdVerified: false }, setup: { path: "action.txt", sha256: null, published: false }, attached: null }; } +function readiness() { return { ready: true, binding, root, runtime: { dev: "1", ino: "4", runnerExecutableSha256: hash("runnerd") } }; } function harness() { let lease: Record = { id: "lease", companyId: "company", environmentId: "environment", heartbeatRunId: "run", provider: "daytona", providerLeaseId: "sandbox", status: "active", releasedAt: null, metadata: { sandboxId: "sandbox", image: authority.image, reuseLease: false, remoteCwd: "/workspace", workspaceSentinel: { path: "/workspace/.paperclip-runtime/reusable-sandbox-lease.json", token: "fixture-sentinel-token", result: "written", runId: "run", providerLeaseId: "sandbox" } } }; @@ -33,7 +38,7 @@ function harness() { const request = JSON.parse(Buffer.from(encoded, "base64").toString()); calls.push({ command, request, timeout }); if (request.op === "wait") return { exitCode: 0, result: JSON.stringify({ ok: true, result: await terminal }) }; if (override) { const result = override(request); if (result !== undefined) return result as { exitCode: number; result: string }; } - let result: unknown = structuredClone(current); + let result: unknown = request.op === "runtime-ready" ? readiness() : structuredClone(current); if (request.op === "publish") { current.setup = { path: request.path, published: true, sha256: hash(request.text) }; result = current.setup; } if (request.op === "close") result = { closed: true }; if (request.op === "arm") result = { armed: true, sealed: false }; @@ -72,10 +77,101 @@ describe("remote native lease admission", () => { const h = harness(); h.labels["paperclip-run-id"] = "other"; await expect(bindRemoteNativeFixture(h.options)).rejects.toThrow("sandbox_labels"); expect(h.get).toHaveBeenCalledExactlyOnceWith("sandbox"); expect(h.executeCommand).not.toHaveBeenCalled(); }); + it("waits beyond the old 20s install subdeadline for staging, then installs exactly once before action", async () => { + vi.useFakeTimers(); vi.setSystemTime(1_000_000); + try { + const h = harness(); h.options.deadlineAt += 60_000; + h.override(r => r.op === "runtime-ready" && Date.now() < 1_021_000 ? { exitCode: 0, result: JSON.stringify({ ok: true, result: { ready: false } }) } : undefined); + const pending = bindRemoteNativeFixture(h.options); + await vi.advanceTimersByTimeAsync(20_000); + expect(h.calls.every(c => c.request.op === "runtime-ready")).toBe(true); + await vi.advanceTimersByTimeAsync(1000); const fixture = await pending; + expect(h.calls.filter(c => c.request.op === "install")).toHaveLength(1); + expect(h.calls.find(c => c.request.op === "install")!.timeout).toBe(25); + expect(h.calls.at(-1)!.request.op).toBe("arm"); + expect(h.apiGet).toHaveBeenCalledTimes((h.calls.filter(c => c.request.op === "runtime-ready").length + 2) * 2); + await fixture.close(); + } finally { vi.useRealTimers(); } + }); + it("never installs on a missing runtime and reserves installation and teardown inside the original deadline", async () => { + vi.useFakeTimers(); vi.setSystemTime(1_000_000); + try { + const h = harness(); h.override(r => r.op === "runtime-ready" ? { exitCode: 0, result: '{"ok":true,"result":{"ready":false}}' } : undefined); + const outcome = bindRemoteNativeFixture(h.options).catch(error => error); + await vi.advanceTimersByTimeAsync(18_000); + expect((await outcome).message).toContain("readiness_deadline"); + expect(h.calls.every(c => c.request.op === "runtime-ready")).toBe(true); + expect(Date.now()).toBe(h.options.deadlineAt - 42_000); + } finally { vi.useRealTimers(); } + }); + it.each(["lease", "binding", "shape", "pin", "unknown-error"])("rejects %s during readiness without installing or publishing", async variant => { + const h = harness(); + h.override(r => { + if (r.op !== "runtime-ready") return undefined; + if (variant === "lease") { h.setLease({ ...h.lease(), heartbeatRunId: "other" }); return { exitCode: 0, result: '{"ok":true,"result":{"ready":false}}' }; } + if (variant === "unknown-error") return { exitCode: 2, result: 'SECRET sdk stderr' }; + const value: any = readiness(); + if (variant === "binding") value.binding = { ...binding, leaseId: "foreign" }; + if (variant === "pin") value.runtime.runnerExecutableSha256 = hash("foreign"); + if (variant === "shape") value.extra = "secret"; + return { exitCode: 0, result: JSON.stringify({ ok: true, result: value }) }; + }); + const error = await bindRemoteNativeFixture(h.options).catch(error => error); + expect(error).toBeInstanceOf(Error); + expect(h.calls.every(c => c.request.op === "runtime-ready")).toBe(true); + expect(JSON.stringify(remoteNativeFixtureDiagnostics(error))).not.toContain("SECRET"); + }); + it.each(["extra", "foreign-phase", "unknown-code", "malformed"])("drops %s remote diagnostic content without retry", async variant => { + const h = harness(); + h.override(r => r.op === "runtime-ready" ? { exitCode: 2, result: variant === "malformed" ? "PRIVATE stderr" : JSON.stringify({ ok: false, diagnostic: { phase: variant === "foreign-phase" ? "close" : r.op, code: variant === "unknown-code" ? "PRIVATE stderr" : "socket_error", ...(variant === "extra" ? { private: "PRIVATE stderr" } : {}) } }) } : undefined); + const error = await bindRemoteNativeFixture(h.options).catch(error => error); + expect(remoteNativeFixtureDiagnostics(error)).toEqual([{ phase: "runtime-ready", code: "invalid_response" }]); + expect(error.message).not.toContain("PRIVATE"); expect(h.calls).toHaveLength(1); + }); + it("persists safe startup diagnostics through the real bootstrap catch without publishing an action", async () => { + const h = harness(), evidence = vi.fn(async () => {}); + h.override(r => r.op === "runtime-ready" ? { exitCode: 2, result: JSON.stringify({ ok: false, diagnostic: { phase: "runtime-ready", code: "runtime_binary_identity" } }) } : undefined); + const bootstrap = createRemoteNativeBootstrap({ ...h.options, companyId: "company", environmentId: "environment", agentId: "agent", image: authority.image, evidence, + api: { get: (async (path: string) => path === "/api/issues/issue" ? { id: "issue", companyId: "company", assigneeAgentId: "agent" } + : path === "/api/heartbeat-runs/run" ? { id: "run", companyId: "company", agentId: "agent", status: "running", executionStage: "preparing" } + : [{ ...h.lease(), issueId: "issue" }]) as RemoteNativeFixtureOptions["api"]["get"] }, + }, async options => bindRemoteNativeFixture({ ...h.options, actionFile: options.actionFile })); + bootstrap.prompt("nonce"); + const actionPrompt = vi.fn(async () => "PRIVATE ACTION"); + await expect(bootstrap.bindAndRelease({ issueId: "issue", runId: "run", targets: ["result.txt"], actionPrompt })).rejects.toThrow("runtime_binary_identity"); + expect(actionPrompt).not.toHaveBeenCalled(); + expect(evidence).toHaveBeenCalledWith("remote-native-bootstrap-startup-run.json", expect.objectContaining({ + phase: "observer_setup", fixtureDiagnostics: [{ phase: "runtime-ready", code: "runtime_binary_identity" }], + })); + expect(JSON.stringify(evidence.mock.calls)).not.toContain("PRIVATE ACTION"); + expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready"]); + }); + it.each(["pid", "ppid", "startTicks", "bootId"])("rejects changed %s between readiness and baseline despite extra observer process fields", async key => { + const h = harness(); + const changed = { state: "S", group: 21, ...root, [key]: key === "bootId" ? "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" : key === "startTicks" ? "999" : 99 }; + h.current.processes = { captured: true, root: changed, journal: [changed], live: [changed.pid] }; + await expect(bindRemoteNativeFixture(h.options)).rejects.toThrow("runtime_identity_changed"); + expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "close"]); + }); + it("retains the closed read phase on transport errors without leaking text", async () => { + const h = harness(), f = await bindRemoteNativeFixture(h.options); + h.override(r => r.op === "read" ? { exitCode: 2, result: JSON.stringify({ ok: false, diagnostic: { phase: "read", code: "socket_error" } }) } : undefined); + const error = await f.readFile("result.txt").catch(error => error); + expect(remoteNativeFixtureDiagnostics(error)).toEqual([{ phase: "read", code: "socket_error" }]); + await f.close(); + }); + it("retains only allowlisted install and cleanup diagnostics with no uncertain retry", async () => { + const h = harness(); + h.override(r => ["install", "close"].includes(r.op) ? { exitCode: 2, result: JSON.stringify({ ok: false, diagnostic: { phase: r.op, code: r.op === "install" ? "runtime_identity_changed" : "socket_error" } }) } : undefined); + const error = await bindRemoteNativeFixture(h.options).catch(error => error); + expect(error.message).toContain("startup_failed_cleanup_unproven"); + expect(remoteNativeFixtureDiagnostics(error)).toEqual([{ phase: "close", code: "socket_error" }, { phase: "install", code: "runtime_identity_changed" }]); + expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "close"]); + }); it("arms before publish, binds long receipt before teardown and preserves exact final bytes", async () => { const h = harness(), f = await bindRemoteNativeFixture(h.options); - expect(h.calls.map(c => c.request.op)).toEqual(["install", "wait", "arm"]); expect(f.baseline.processes.live).toEqual([21]); - expect(h.calls[1]!.timeout).toBeLessThanOrEqual(45); expect(h.calls[1]!.request.timeoutMs).toBe(h.calls[1]!.timeout! * 1000); + expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "wait", "arm"]); expect(f.baseline.processes.live).toEqual([21]); + expect(h.calls[2]!.timeout).toBeLessThanOrEqual(45); expect(h.calls[2]!.request.timeoutMs).toBe(h.calls[2]!.timeout! * 1000); await f.publishAction("action.txt", "write only result.txt"); await expect(f.publishAction("action.txt", "retry")).rejects.toThrow("publish_bound"); const bytes = "\nUnicode 🪴 literal \\n\n"; @@ -83,7 +179,7 @@ describe("remote native lease admission", () => { final.targets["result.txt"] = { ...final.targets["result.txt"]!, absent: false, sha256: hash(bytes), mutationCount: 1 }; h.resolveTerminal(final); h.apiGet.mockRejectedValue(new Error("lease already deleted")); expect((await f.finish()).processes.live).toEqual([]); expect((await f.readFile("result.txt")).toString()).toBe(bytes); - await f.close(); expect(h.calls.map(c => c.request.op)).toEqual(["install", "wait", "arm", "publish"]); + await f.close(); expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "wait", "arm", "publish"]); }); it("fails closed when lease deletion beats the terminal receipt", async () => { const h = harness(), f = await bindRemoteNativeFixture(h.options); await f.publishAction("action.txt", "task"); @@ -115,12 +211,12 @@ describe("remote native lease admission", () => { it("cleans only its admitted observer on a failed startup receipt and never publishes", async () => { const h = harness(); h.current.processes = { captured: false, root: null, journal: [], live: [] }; await expect(bindRemoteNativeFixture(h.options)).rejects.toThrow("bootstrap_not_held"); - expect(h.calls.map(c => c.request.op)).toEqual(["install", "close"]); + expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "close"]); }); it("refuses action publication without a confirmed receipt channel", async () => { const h = harness(); h.override(r => r.op === "arm" ? { exitCode: 0, result: JSON.stringify({ ok: true, result: { armed: false, sealed: false } }) } : undefined); await expect(bindRemoteNativeFixture(h.options)).rejects.toThrow("receipt_channel_not_armed"); - expect(h.calls.map(c => c.request.op)).toEqual(["install", "wait", "arm", "close"]); + expect(h.calls.map(c => c.request.op)).toEqual(["runtime-ready", "install", "wait", "arm", "close"]); }); it("bounds malformed command output and does not replay an uncertain publish", async () => { const h = harness(), f = await bindRemoteNativeFixture(h.options); @@ -131,12 +227,12 @@ describe("remote native lease admission", () => { }); it("ships syntactically valid closed Node programs with exact binary and no provider env", async () => { const h = harness(); await bindRemoteNativeFixture(h.options); - const install = h.calls[0]!; const source = install.request.source as string; + const install = h.calls.find(c => c.request.op === "install")!; const source = install.request.source as string; expect(() => new Script(source)).not.toThrow(); expect(source).not.toContain("__name("); const rpcQuoted = install.command.match(/ -e (.+) '[A-Za-z0-9+/=]+'$/su)![1]!; const rpc = rpcQuoted.slice(1, -1).replaceAll("'\\''", "'"); expect(() => new Script(rpc)).not.toThrow(); - expect(rpc).toContain("startedAt+20000"); expect(install.timeout).toBeLessThanOrEqual(27); expect(rpc).toContain("r.timeoutMs-(Date.now()-startedAt)"); + expect(rpc).not.toContain("startedAt+20000"); expect(install.timeout).toBeLessThanOrEqual(27); expect(rpc).toContain("r.timeoutMs-(Date.now()-startedAt)"); expect(source).toContain("/proc/"); expect(source).toContain("workspaceWatch"); expect(source).toContain("finalReceipt.files"); expect(install.command).toMatch(/^\/usr\/bin\/env -i PATH=\/usr\/bin:\/bin /u); expect(install.request.config.runnerdSha256).toBe(hash("runnerd")); @@ -220,9 +316,14 @@ describe("independent filesystem and process observations", () => { }); describe("actual generated observer state machine", () => { - async function observerHarness(deferStartup = false) { + async function observerHarness(deferStartup = false, transformed?: { command: string; observer: string }) { const h = harness(); await bindRemoteNativeFixture(h.options); - const { source, config } = h.calls[0]!.request; + const install = h.calls.find(c => c.request.op === "install")!; + if (transformed) { + install.request.source = transformed.observer; + install.command = transformed.command; + } + const { source, config } = install.request; const intervals: Array<() => void> = [], timers: Array<{ fn: () => void; ms: number }> = []; const proc = new Map([[21, { ppid: 1, group: 21, ticks: "100", argv: ["/workspace/.paperclip-runtime/paperclip-runner/bin/paperclip-runnerd", "--run-id", "run", "--environment-lease-id", "workspace-id", "--lifecycle-mode", "per_turn", "--state-dir", "/workspace/.paperclip-runtime/paperclip-runner/sessions/" + "a".repeat(64) + "/runner"] }]]); const files = new Map([[`${config.root}/observer.cjs`, Buffer.from(source)], [config.sentinel.path, Buffer.from(JSON.stringify({ version: 1, provider: "daytona", token: config.sentinel.token, companyId: "company", environmentId: "environment" }))]]); @@ -297,7 +398,7 @@ describe("actual generated observer state machine", () => { const replies: any[] = [], socket = Object.assign(new EventEmitter(), { end: (value: string) => replies.push(JSON.parse(value)), destroy: vi.fn() }); handlers[0]!(socket); socket.emit("data", Buffer.from(JSON.stringify({ op, nonce: config.nonce, ...args }) + "\n")); return replies; } - return { request, proc, files, watches, fs, intervals, timers, config, handlers, children, symbolicLinks, context, server, directories, listeners, install: h.calls[0]!, replaceRuntimeRoot() { runtimeInode = 999n; } }; + return { request, proc, files, watches, fs, intervals, timers, config, handlers, children, symbolicLinks, context, server, directories, listeners, install: h.calls.find(c => c.request.op === "install")!, replaceRuntimeRoot() { runtimeInode = 999n; } }; } async function generatedRpc(o: Awaited>, request: Record, mutateSource = (source: string) => source) { const quoted = o.install.command.match(/ -e (.+) '[A-Za-z0-9+/=]+'$/su)![1]!; @@ -341,6 +442,72 @@ describe("actual generated observer state machine", () => { await new Promise(resolve => setImmediate(resolve)); return { output, process, forwarded, spawn, connections }; } + it("executes readiness and installation after the actual pinned tsx production transform", async () => { + const modulePath = fileURLToPath(new URL("./remote-native-fixtures.ts", import.meta.url)); + const loaderPath = fileURLToPath(new URL("../../cli/node_modules/tsx/dist/loader.mjs", import.meta.url)); + const script = ` + import { bindRemoteNativeFixture } from ${JSON.stringify(modulePath)}; + const binding = ${JSON.stringify(binding)}, ready = ${JSON.stringify(readiness())}; + const lease = { id: binding.leaseId, companyId: binding.companyId, environmentId: binding.environmentId, heartbeatRunId: binding.runId, + provider: 'daytona', providerLeaseId: binding.sandboxId, status: 'active', releasedAt: null, + metadata: { sandboxId: binding.sandboxId, image: binding.image, reuseLease: false, remoteCwd: binding.remoteCwd, + workspaceSentinel: { path: binding.remoteCwd+'/.paperclip-runtime/reusable-sandbox-lease.json', token: 'fixture-sentinel-token', result: 'written', runId: binding.runId, providerLeaseId: binding.sandboxId } } }; + let programs; + const sandbox = { id: binding.sandboxId, labels: { 'paperclip-provider':'daytona','paperclip-company-id':binding.companyId,'paperclip-environment-id':binding.environmentId,'paperclip-run-id':binding.runId,'paperclip-reuse-lease':'false' }, + process: { async executeCommand(command) { + const encoded = command.slice(command.lastIndexOf(" '")+2,-1), request = JSON.parse(Buffer.from(encoded,'base64')); + if(request.op==='runtime-ready') return {exitCode:0,result:JSON.stringify({ok:true,result:ready})}; + if(request.op==='install') programs = { command, observer:request.source }; + return {exitCode:0,result:JSON.stringify({ok:true,result:request.op==='close'?{closed:true}:{}})}; + } } }; + try { await bindRemoteNativeFixture({ authority: ${JSON.stringify(authority)}, api:{get:async p=>p.includes('/environments/')?[lease]:lease}, daytona:{get:async()=>sandbox}, + sdkVersion:'0.203.0',nodeSha256:${JSON.stringify(hash("node"))},runnerdSha256:${JSON.stringify(hash("runnerd"))},targets:['result.txt'],actionFile:'action.txt',deadlineAt:Date.now()+60000 }); } catch {} + if(!programs) throw Error('No captured generated programs'); + process.stdout.write(JSON.stringify(programs)); + `; + const transformed = JSON.parse(execFileSync(process.execPath, ["--import", loaderPath, "--input-type=module", "-e", script], { + env: { PATH: "/usr/bin:/bin", HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, TSX_DISABLE_CACHE: "1" }, timeout: 10_000, maxBuffer: 256 * 1024, encoding: "utf8", + })); + expect(transformed.command).not.toContain("__name("); expect(transformed.observer).not.toContain("__name("); + const o = await observerHarness(true, transformed); + const ready = await generatedRpc(o, { ...o.install.request, op: "runtime-ready" }); + expect(JSON.parse(ready.output)).toEqual({ ok: true, result: readiness() }); + const installed = await generatedRpc(o, o.install.request); + expect(installed.process.exitCode).toBe(0); expect(JSON.parse(installed.output).result.complete).toBe(true); + const closed = await generatedRpc(o, { op: "close", root: o.config.root, nonce: o.config.nonce, nodeSha256: hash("node"), timeoutMs: 10_000 }); + expect(JSON.parse(closed.output)).toEqual({ ok: true, result: { closed: true } }); + o.timers.find(t => t.ms === 100)!.fn(); expect(o.watches.every(w => w.closed)).toBe(true); + }); + it("executes the actual readiness probe without creating an observer, then revalidates its exact identity at install", async () => { + const o = await observerHarness(true); + const ready = await generatedRpc(o, { ...o.install.request, op: "runtime-ready" }); + expect(JSON.parse(ready.output)).toEqual({ ok: true, result: readiness() }); + expect(ready.spawn).not.toHaveBeenCalled(); expect(o.fs.mkdirSync).not.toHaveBeenCalled(); expect(o.watches).toHaveLength(0); + o.replaceRuntimeRoot(); + const changed = await generatedRpc(o, o.install.request); + expect(JSON.parse(changed.output).diagnostic.code).toBe("runtime_identity_changed"); + expect(changed.spawn).not.toHaveBeenCalled(); expect(o.fs.mkdirSync).not.toHaveBeenCalled(); + }); + it.each(["absent", "foreign-run", "foreign-group", "ambiguous", "wrong-binary", "symlink", "disappeared"])("checks generated runtime readiness: %s", async variant => { + const o = await observerHarness(true), runtimePath = "/workspace/.paperclip-runtime/paperclip-runner"; + if (variant === "absent") o.directories.delete(runtimePath); + if (variant === "foreign-run") o.proc.get(21)!.argv[2] = "foreign"; + if (variant === "foreign-group") o.proc.get(21)!.group = 99; + if (variant === "ambiguous") o.proc.set(22, { ...o.proc.get(21)!, group: 22 }); + if (variant === "symlink") o.symbolicLinks.add(runtimePath); + if (variant === "disappeared") o.proc.clear(); + const request = { ...o.install.request, op: variant === "disappeared" ? "install" : "runtime-ready", config: { ...o.config, ...(variant === "wrong-binary" ? { runnerdSha256: hash("wrong") } : {}) } }; + const result = await generatedRpc(o, request), parsed = JSON.parse(result.output); + if (["absent", "foreign-run", "foreign-group"].includes(variant)) expect(parsed).toEqual({ ok: true, result: { ready: false } }); + else expect(parsed).toEqual({ ok: false, diagnostic: { phase: request.op, code: { ambiguous: "ambiguous_run_root", "wrong-binary": "runtime_binary_identity", symlink: "runtime_root_identity", disappeared: "runtime_not_ready" }[variant] } }); + expect(result.spawn).not.toHaveBeenCalled(); expect(o.fs.mkdirSync).not.toHaveBeenCalled(); + }); + it("uses the closed read phase for actual generated socket errors", async () => { + const o = await observerHarness(true); + const read = await generatedRpc(o, { op: "read", root: o.config.root, nonce: o.config.nonce, nodeSha256: hash("node"), timeoutMs: 10_000, path: "result.txt" }); + expect(JSON.parse(read.output)).toEqual({ ok: false, diagnostic: { phase: "read", code: "socket_error" } }); + expect(read.spawn).not.toHaveBeenCalled(); + }); it("executes generated install through the bounded observer socket, then closes the exact observer", async () => { const o = await observerHarness(true); expect(Buffer.byteLength(o.install.request.source)).toBeGreaterThan(8192); @@ -348,7 +515,13 @@ describe("actual generated observer state machine", () => { expect(installed.process.exitCode).toBe(0); const baseline = JSON.parse(installed.output); expect(baseline.ok).toBe(true); expect(baseline.result.complete).toBe(true); - expect(baseline.result.processes.root.pid).toBe(21); + expect(baseline.result.processes.root).toMatchObject({ ...root, group: 21, state: "S" }); + // Feed the actual generated observer receipt through the real host reader, + // including parseRemoteProcStat's extra fields and different key order. + const h = harness(); h.override(request => request.op === "install" ? { exitCode: 0, result: installed.output } : undefined); + const fixture = await bindRemoteNativeFixture(h.options); + expect(fixture.baseline.processes.root).toEqual(baseline.result.processes.root); + await fixture.close(); expect(installed.spawn).toHaveBeenCalledTimes(1); expect(installed.forwarded.map(value => JSON.parse(value))).toEqual([{ op: "snapshot", nonce: o.config.nonce }]); expect(Buffer.byteLength(installed.forwarded[0]!)).toBeLessThan(8192); @@ -369,7 +542,7 @@ describe("actual generated observer state machine", () => { const mutant = source.replace("fs.mkdirSync(c.root,{mode:0o700});", ""); expect(mutant).not.toBe(source); return mutant; }); - expect(installed.process.exitCode).toBe(2); expect(installed.output).toBe(""); + expect(installed.process.exitCode).toBe(2); expect(JSON.parse(installed.output).diagnostic).toEqual({ phase: "install", code: "remote_unknown" }); expect(installed.spawn).not.toHaveBeenCalled(); expect(installed.connections).toHaveLength(0); expect(o.files.has(`${o.config.root}/observer.cjs`)).toBe(false); expect(o.directories.has(o.config.root)).toBe(false); expect(o.watches).toHaveLength(0); @@ -380,7 +553,7 @@ describe("actual generated observer state machine", () => { const mutant = source.replace("server.listen(path.join(config.root,'control.sock'));", ""); expect(mutant).not.toBe(source); const installed = await generatedRpc(o, { ...o.install.request, source: mutant }); - expect(installed.process.exitCode).toBe(2); expect(installed.output).toBe(""); + expect(installed.process.exitCode).toBe(2); expect(JSON.parse(installed.output).diagnostic).toEqual({ phase: "install", code: "socket_error" }); expect(installed.spawn).toHaveBeenCalledTimes(1); expect(o.handlers).toHaveLength(1); expect(o.server.listen).not.toHaveBeenCalled(); expect(o.listeners.size).toBe(0); expect(installed.connections).toHaveLength(0); expect(installed.forwarded).toHaveLength(0); @@ -393,7 +566,7 @@ describe("actual generated observer state machine", () => { it("still rejects an oversized control request and can close its observer afterward", async () => { const o = await observerHarness(); const oversized = await generatedRpc(o, { op: "snapshot", root: o.config.root, nonce: o.config.nonce, nodeSha256: hash("node"), timeoutMs: 10_000, unexpected: "x".repeat(8192) }); - expect(oversized.process.exitCode).toBe(2); expect(oversized.output).toBe(""); + expect(oversized.process.exitCode).toBe(2); expect(JSON.parse(oversized.output).diagnostic).toEqual({ phase: "snapshot", code: "socket_error" }); expect(oversized.connections[0]!.observer.destroy).toHaveBeenCalled(); const closed = await generatedRpc(o, { op: "close", root: o.config.root, nonce: o.config.nonce, nodeSha256: hash("node"), timeoutMs: 10_000 }); expect(JSON.parse(closed.output)).toEqual({ ok: true, result: { closed: true } }); diff --git a/tests/runner-e2e/remote-native-fixtures.ts b/tests/runner-e2e/remote-native-fixtures.ts index 150cee5bae..47b4685c71 100644 --- a/tests/runner-e2e/remote-native-fixtures.ts +++ b/tests/runner-e2e/remote-native-fixtures.ts @@ -22,6 +22,23 @@ function relative(value: string): string { fail(value.length <= 240 && /^[a-zA-Z0-9._/-]+$/u.test(value) && !value.startsWith("/") && value.split("/").every(part => part !== "" && part !== "." && part !== ".."), "unsafe_relative_target"); return value; } +// Only closed diagnostic enums cross the remote boundary; SDK errors and output +// are never retained. These diagnostics explain failed evidence, not qualification. +const RPC_PHASES = ["runtime-ready", "install", "wait", "close", "arm", "publish", "snapshot", "attached", "read"] as const; +const RPC_CODES = ["node_identity", "sentinel_type", "sentinel", "cwd", "runtime_root_identity", "runtime_binary_identity", "proc_bound", "ambiguous_run_root", "runtime_not_ready", "runtime_identity_changed", "invalid_proc_identity", "invalid_proc_fields", "socket_error", "socket_timeout", "output_bound", "rpc_deadline", "remote_unknown", "transport_failure", "invalid_response", "readiness_deadline"] as const; +type RpcPhase = typeof RPC_PHASES[number]; +type RpcCode = typeof RPC_CODES[number]; +interface RpcDiagnostic { phase: RpcPhase; code: RpcCode } +class RemoteFixtureError extends Error { + constructor(message: string, readonly diagnostic: RpcDiagnostic, options?: ErrorOptions) { super(message, options); } +} +export function remoteNativeFixtureDiagnostics(error: unknown): RpcDiagnostic[] { + const result: RpcDiagnostic[] = []; + for (let current = error, depth = 0; current instanceof Error && depth < 3; current = current.cause, depth++) { + if (current instanceof RemoteFixtureError) result.push({ ...current.diagnostic }); + } + return result; +} export interface RemoteNativeAuthority { companyId: string; environmentId: string; runId: string; leaseId: string; sandboxId: string; image: string } export interface RemoteNativeBinding extends RemoteNativeAuthority { remoteCwd: string } export interface RemoteProcessIdentity { pid: number; ppid: number; startTicks: string; bootId: string } @@ -167,15 +184,18 @@ function observerSource() { .replace("WATCH_TARGET", () => createRemoteTargetWatch.toString()).replace("ATTACHED_CLIENT", () => JSON.stringify(ATTACHED_CLIENT)); } const RPC = String.raw`const fs=require('node:fs'),net=require('node:net'),cp=require('node:child_process'),crypto=require('node:crypto');const r=JSON.parse(Buffer.from(process.argv[1],'base64').toString());const hash=x=>'sha256:'+crypto.createHash('sha256').update(x).digest('hex'); -const startedAt=Date.now();if(!Number.isInteger(r.timeoutMs)||r.timeoutMs<1000||r.timeoutMs>300000)throw Error('rpc_deadline');setTimeout(()=>process.exit(2),r.timeoutMs).unref(); +const startedAt=Date.now();if(!Number.isInteger(r.timeoutMs)||r.timeoutMs<1000||r.timeoutMs>300000)throw Error('rpc_deadline');setTimeout(()=>{failure('rpc_deadline');process.exit(2)},r.timeoutMs).unref(); const parseStat=PARSE_STAT,runRoot=RUN_ROOT; -async function waitRuntime(c){const root=c.binding.remoteCwd+'/'+c.runtimeRelative,boot=fs.readFileSync('/proc/sys/kernel/random/boot_id','utf8').trim(),until=Math.min(startedAt+20000,startedAt+r.timeoutMs-1000);while(Date.now()8192)throw Error('proc_bound');for(const name of entries){if(!/^\d+$/.test(name)||Number(name)<2)continue;try{const p=parseStat(Number(name),fs.readFileSync('/proc/'+name+'/stat','utf8'),boot),argv=fs.readFileSync('/proc/'+name+'/cmdline').toString().split('\0').filter(Boolean);if(runRoot(argv,c.binding.runId,p)){if(argv[0]!==root+'/bin/paperclip-runnerd'||hash(fs.readFileSync('/proc/'+name+'/exe'))!==c.runnerdSha256)throw Error('runtime_binary_identity');matches.push(p)}}catch(e){if(e.code!=='ENOENT'&&e.code!=='ESRCH')throw e}}if(matches.length>1)throw Error('ambiguous_run_root');if(matches.length===1)return;}catch(e){if(e.code!=='ENOENT')throw e}await new Promise(resolve=>setTimeout(resolve,50))}throw Error('runtime_not_ready')} -(async()=>{let controlRequest=r;if(hash(fs.readFileSync(process.execPath))!==r.nodeSha256)throw Error('node_identity');if(r.op==='install'){const c=r.config;await waitRuntime(c);const st=fs.lstatSync(c.sentinel.path);if(!st.isFile()||st.isSymbolicLink()||st.size>16384||fs.realpathSync(c.sentinel.path)!==c.sentinel.path)throw Error('sentinel_type');const s=JSON.parse(fs.readFileSync(c.sentinel.path,'utf8'));if(s.version!==1||s.provider!=='daytona'||s.token!==c.sentinel.token||s.companyId!==c.binding.companyId||s.environmentId!==c.binding.environmentId)throw Error('sentinel');if(fs.realpathSync(c.binding.remoteCwd)!==c.binding.remoteCwd)throw Error('cwd');fs.mkdirSync(c.root,{mode:0o700});fs.writeFileSync(c.root+'/observer.cjs',r.source,{flag:'wx',mode:0o400});c.observerTtlMs=Math.max(1,c.observerTtlMs-(Date.now()-startedAt));const child=cp.spawn(process.execPath,[c.root+'/observer.cjs',Buffer.from(JSON.stringify(c)).toString('base64')],{detached:true,stdio:'ignore',env:{PATH:'/usr/bin:/bin'}});child.unref();r.root=c.root;controlRequest={op:'snapshot',nonce:c.nonce};} -for(let i=0;!fs.existsSync(r.root+'/control.sock')&&i<200;i++)await new Promise(resolve=>setTimeout(resolve,10));const socket=net.connect(r.root+'/control.sock');let output='';socket.setTimeout(Math.max(1,r.timeoutMs-(Date.now()-startedAt)));socket.on('timeout',()=>{socket.destroy();process.exitCode=2});socket.on('error',()=>{process.exitCode=2});socket.on('connect',()=>socket.write(JSON.stringify(controlRequest)+'\n'));socket.on('data',b=>{output+=b;if(Buffer.byteLength(output)>262144){socket.destroy();process.exitCode=2}});socket.on('end',()=>{if(!process.exitCode)process.stdout.write(output)}); -})().catch(()=>{process.exitCode=2});`; +const codes=RPC_CODES;let failed=false;const phase=RPC_PHASES.includes(r.op)?r.op:'install'; +function failure(code){if(failed)return;failed=true;process.exitCode=2;process.stdout.write(JSON.stringify({ok:false,diagnostic:{phase,code:codes.includes(code)?code:'remote_unknown'}})+'\n')} +function admitted(c){const st=fs.lstatSync(c.sentinel.path);if(!st.isFile()||st.isSymbolicLink()||st.size>16384||fs.realpathSync(c.sentinel.path)!==c.sentinel.path)throw Error('sentinel_type');const s=JSON.parse(fs.readFileSync(c.sentinel.path,'utf8'));if(s.version!==1||s.provider!=='daytona'||s.token!==c.sentinel.token||s.companyId!==c.binding.companyId||s.environmentId!==c.binding.environmentId)throw Error('sentinel');if(fs.realpathSync(c.binding.remoteCwd)!==c.binding.remoteCwd)throw Error('cwd')} +function runtime(c){const root=c.binding.remoteCwd+'/'+c.runtimeRelative,boot=fs.readFileSync('/proc/sys/kernel/random/boot_id','utf8').trim();let s;try{s=fs.lstatSync(root,{bigint:true})}catch(e){if(e.code==='ENOENT')return {ready:false};throw e}if(!s.isDirectory()||s.isSymbolicLink()||fs.realpathSync(root)!==root)throw Error('runtime_root_identity');const matches=[];const entries=fs.readdirSync('/proc');if(entries.length>8192)throw Error('proc_bound');for(const name of entries){if(!/^\d+$/.test(name)||Number(name)<2)continue;try{const p=parseStat(Number(name),fs.readFileSync('/proc/'+name+'/stat','utf8'),boot),argv=fs.readFileSync('/proc/'+name+'/cmdline').toString().split('\0').filter(Boolean);if(runRoot(argv,c.binding.runId,p)){if(argv[0]!==root+'/bin/paperclip-runnerd'||hash(fs.readFileSync('/proc/'+name+'/exe'))!==c.runnerdSha256)throw Error('runtime_binary_identity');if(p.state!=='Z')matches.push({pid:p.pid,ppid:p.ppid,startTicks:p.startTicks,bootId:p.bootId})}}catch(e){if(e.code!=='ENOENT'&&e.code!=='ESRCH')throw e}}if(matches.length>1)throw Error('ambiguous_run_root');return matches.length===1?{ready:true,binding:c.binding,root:matches[0],runtime:{dev:String(s.dev),ino:String(s.ino),runnerExecutableSha256:c.runnerdSha256}}:{ready:false}} +(async()=>{let controlRequest=r;if(hash(fs.readFileSync(process.execPath))!==r.nodeSha256)throw Error('node_identity');if(r.op==='runtime-ready'||r.op==='install'){const c=r.config;admitted(c);const current=runtime(c);if(r.op==='runtime-ready'){process.stdout.write(JSON.stringify({ok:true,result:current})+'\n');return}if(!current.ready)throw Error('runtime_not_ready');if(JSON.stringify(current)!==JSON.stringify(r.expectedRuntime))throw Error('runtime_identity_changed');fs.mkdirSync(c.root,{mode:0o700});fs.writeFileSync(c.root+'/observer.cjs',r.source,{flag:'wx',mode:0o400});c.observerTtlMs=Math.max(1,c.observerTtlMs-(Date.now()-startedAt));const child=cp.spawn(process.execPath,[c.root+'/observer.cjs',Buffer.from(JSON.stringify(c)).toString('base64')],{detached:true,stdio:'ignore',env:{PATH:'/usr/bin:/bin'}});child.unref();r.root=c.root;controlRequest={op:'snapshot',nonce:c.nonce};} +for(let i=0;!fs.existsSync(r.root+'/control.sock')&&i<200;i++)await new Promise(resolve=>setTimeout(resolve,10));const socket=net.connect(r.root+'/control.sock');let output='';socket.setTimeout(Math.max(1,r.timeoutMs-(Date.now()-startedAt)));socket.on('timeout',()=>{socket.destroy();failure('socket_timeout')});socket.on('error',()=>{failure('socket_error')});socket.on('connect',()=>socket.write(JSON.stringify(controlRequest)+'\n'));socket.on('data',b=>{output+=b;if(Buffer.byteLength(output)>262144){socket.destroy();failure('output_bound')}});socket.on('end',()=>{if(!process.exitCode)process.stdout.write(output)}); +})().catch(e=>{failure(typeof e?.message==='string'?e.message:'remote_unknown')});`; function rpcSource() { - return RPC.replace("PARSE_STAT", () => parseRemoteProcStat.toString()).replace("RUN_ROOT", () => isRemoteRunRoot.toString()); + return RPC.replace("RPC_CODES", () => JSON.stringify(RPC_CODES)).replace("RPC_PHASES", () => JSON.stringify(RPC_PHASES)).replace("PARSE_STAT", () => parseRemoteProcStat.toString()).replace("RUN_ROOT", () => isRemoteRunRoot.toString()); } export interface RemoteNativeFixture { @@ -309,8 +329,8 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption ]); } finally { if (timer) clearTimeout(timer); } } - async function rpc(request: Record, admitted?: Awaited>) { - const available = request.op === "close" ? CLOSE_GRACE_MS : receiptDeadlineAt - Date.now(); + async function rpc(request: Record, admitted?: Awaited>, deadlineAt = receiptDeadlineAt) { + const available = request.op === "close" ? CLOSE_GRACE_MS : deadlineAt - Date.now(); const cap = request.op === "install" ? 27_000 : request.op === "wait" ? 300_000 : 12_000; const budgetMs = Math.floor(Math.min(available, cap) / 1000) * 1000; fail(budgetMs >= 1000, "receipt_deadline"); @@ -328,8 +348,17 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption const command = `/usr/bin/env -i PATH=/usr/bin:/bin ${quote(NODE)} -e ${quote(rpcSource())} ${quote(payload)}`; let response: { exitCode: number; result: string }; try { response = await sandbox.process.executeCommand(command, binding!.remoteCwd, {}, timeoutMs / 1000); } - catch { throw new Error("remote_native_fixture:remote_command_failed_or_deadline"); } - fail(response.exitCode === 0 && typeof response.result === "string" && Buffer.byteLength(response.result) <= MAX_OUTPUT, "command_failed_or_output_bound"); + catch { throw new RemoteFixtureError("remote_native_fixture:remote_command_failed_or_deadline", { phase: request.op as RpcPhase, code: "transport_failure" }); } + if (typeof response.result !== "string" || Buffer.byteLength(response.result) > MAX_OUTPUT) throw new RemoteFixtureError("remote_native_fixture:command_failed_or_output_bound", { phase: request.op as RpcPhase, code: "output_bound" }); + if (response.exitCode !== 0) { + let diagnostic: RpcDiagnostic = { phase: request.op as RpcPhase, code: "invalid_response" }; + try { + const value = record(JSON.parse(response.result)), d = record(value.diagnostic); + if (value.ok === false && Object.keys(value).sort().join() === "diagnostic,ok" && Object.keys(d).sort().join() === "code,phase" + && d.phase === request.op && RPC_CODES.includes(d.code as RpcCode)) diagnostic = { phase: d.phase as RpcPhase, code: d.code as RpcCode }; + } catch { /* Never retain remote text. */ } + throw new RemoteFixtureError(`remote_native_fixture:command_failed_or_output_bound:${diagnostic.phase}:${diagnostic.code}`, diagnostic); + } let parsed: Record; try { parsed = record(JSON.parse(response.result)); } catch { throw new Error("remote_native_fixture:invalid_observer_json"); } fail(parsed.ok === true, "observer_incomplete"); @@ -339,16 +368,39 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption const sandbox = await bounded(admittedSandbox, Math.min(10_000, receiptDeadlineAt - Date.now())); const names = [...targets, ...(options.crossRoot ? ["@cross-root"] : [])]; const config = { root, nonce, binding, sentinel, targets, actionFile, crossRoot: options.crossRoot, runtimeRelative: RUNTIME_RELATIVE, runnerdSha256: options.runnerdSha256 }; + // Lease activation precedes runner artifact staging. Observe the exact pinned + // run root before spending the single installation budget. No observer or + // action exists during this polling phase, and failures are never retried. + const readyDeadline = receiptDeadlineAt - 27_000; + let expectedRuntime: Record; + while (true) { + if (readyDeadline - Date.now() < 1000) throw new RemoteFixtureError("remote_native_fixture:readiness_deadline", { phase: "runtime-ready", code: "readiness_deadline" }); + const value = record(await rpc({ op: "runtime-ready", config }, undefined, readyDeadline)); + if (value.ready === true) { + const process = record(value.root), runtime = record(value.runtime); + fail(Object.keys(value).sort().join() === "binding,ready,root,runtime" && JSON.stringify(value.binding) === JSON.stringify(binding) + && Object.keys(process).sort().join() === "bootId,pid,ppid,startTicks" && Number.isSafeInteger(process.pid) && Number(process.pid) > 1 + && Number.isSafeInteger(process.ppid) && Number(process.ppid) >= 0 && typeof process.startTicks === "string" && /^\d+$/u.test(process.startTicks) + && typeof process.bootId === "string" && /^[a-f0-9-]{36}$/u.test(process.bootId) + && Object.keys(runtime).sort().join() === "dev,ino,runnerExecutableSha256" && typeof runtime.dev === "string" && /^\d+$/u.test(runtime.dev) + && typeof runtime.ino === "string" && /^\d+$/u.test(runtime.ino) && runtime.runnerExecutableSha256 === options.runnerdSha256, "runtime_ready_identity"); + expectedRuntime = value; break; + } + fail(value.ready === false && Object.keys(value).length === 1, "runtime_ready_shape"); + await new Promise(resolve => setTimeout(resolve, Math.min(200, Math.max(0, readyDeadline - Date.now())))); + } let baseline: RemoteNativeSnapshot; try { - baseline = readSnapshot(await rpc({ op: "install", config, source: observerSource() }, sandbox), binding!, names, actionFile, options.runnerdSha256); + baseline = readSnapshot(await rpc({ op: "install", config, expectedRuntime, source: observerSource() }), binding!, names, actionFile, options.runnerdSha256); fail(baseline.complete && baseline.processes.captured && baseline.processes.live.length > 0 && baseline.watcher.complete && !baseline.setup.published, "bootstrap_not_held"); + fail((["pid", "ppid", "startTicks", "bootId"] as const).every(key => baseline.processes.root?.[key] === record(expectedRuntime.root)[key]) + && baseline.scope.excludedRuntime.dev === record(expectedRuntime.runtime).dev && baseline.scope.excludedRuntime.ino === record(expectedRuntime.runtime).ino, "runtime_identity_changed"); } catch (error) { // Only the nonce/inode-bound observer can acknowledge this cleanup. If // launch failed before its socket became available, TTL remains a bound, // not a claimed successful cleanup receipt. try { await rpc({ op: "close" }, sandbox); } - catch { throw new Error("remote_native_fixture:startup_failed_cleanup_unproven", { cause: error }); } + catch (cleanupError) { throw new RemoteFixtureError("remote_native_fixture:startup_failed_cleanup_unproven", remoteNativeFixtureDiagnostics(cleanupError)[0] ?? { phase: "close", code: "remote_unknown" }, { cause: error }); } throw error; } // Start receiving while the lease is still authorized, before publishAction.