Add supported remote ingestion fixture and strict provenance gates

scripts/compare-native-ingestion-browser.ts

@@ -17,12 +17,50 @@ const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ bas
 assert.ok(manifest.length >= 5, "At least five actual paired application runs are required");
 const load = async (file: string) => JSON.parse(await readFile(path.resolve(path.dirname(manifestPath), file), "utf8"));
 const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await load(entry.baselineFile), candidate: await load(entry.candidateFile) })));
-const names = ["paced", "burst16", "burst128", "burst512", "existing-history", "operator-stop"];
+const workloads = [
+  { name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false },
+  { name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false },
+  { name: "burst128", runs: 10, events: 128, intervalMs: 0, prelude: 0, stop: false },
+  { name: "burst512", runs: 100, events: 512, intervalMs: 0, prelude: 0, stop: false },
+  { name: "existing-history", runs: 10, events: 128, intervalMs: 0, prelude: 1000, stop: false },
+  { name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true },
+];
+const names = workloads.map(item => item.name);
 const median = (values: number[]) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]!;
 const first = pairs[0];
+const sha256 = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+const uuid = (value: unknown) => typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value);
 for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) {
   const report = pair[arm];
-  assert.equal(report.protocolVersion, "v2-compact-markers-original-clock-stream", "Fresh browser acceptance requires the qualified v2 clock protocol");
+  assert.equal(report.protocolVersion, "v3-supported-remote-original-clock-stream", "Fresh browser acceptance requires the qualified v3 remote clock protocol");
+  assert.equal(report.transport, "sandbox-remote", "V3 requires the supported sandbox remote transport");
+  assert.deepEqual(report.resourceBudgets, { providerReadinessMs: 300_000, fullInvocationMs: 900_000,
+    streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 },
+  "Keep the prospectively registered remote setup/resource budgets and existing streaming/control timeouts");
+  const remote = report.remote;
+  assert.equal(remote?.transport, "sandbox-remote", "Missing normal remote runtime provenance");
+  assert.equal(remote?.plugin?.pluginKey, "paperclip.fake-sandbox-provider", "Missing normal remote plugin provenance");
+  assert.equal(remote?.selection, "Normal plugin install + sandbox environment + agent defaultEnvironmentId; no runtime resolver override",
+    "Use normal API remote environment selection");
+  assert.ok(uuid(remote.environmentId) && uuid(report.companyId), "Missing actual remote company/environment identity");
+  assert.equal(remote.revision, report.sourceRevision);
+  assert.equal(remote.buildRevision, report.compiledBuildRevision);
+  assert.equal(remote.tls?.peerVerification, true, "Remote TLS peer verification is required");
+  assert.equal(typeof remote.runnerPublicUrl, "string", "Missing actual remote WSS endpoint");
+  const endpoint = new URL(remote.runnerPublicUrl);
+  assert.equal(endpoint.protocol, "wss:", "Actual remote endpoint must use verified WSS");
+  assert.equal(endpoint.hostname, "127.0.0.1", "Use the owned loopback remote fixture endpoint");
+  assert.ok(Number(endpoint.port) > 0 && !endpoint.username && !endpoint.password && !endpoint.search && !endpoint.hash,
+    "Use the recorded owned remote fixture endpoint without overrides");
+  for (const value of [remote.tls?.caSha256, remote.tls?.serverCertificateSha256, remote.runner?.sha256,
+    remote.provider?.nodeSha256, remote.provider?.wrapperSha256, remote.provider?.fixtureSha256]) {
+    assert.ok(sha256(value), "Missing remote TLS, daemon or provider artifact fingerprint");
+  }
+  assert.equal(remote.provider.fixtureSha256, report.fixtureSha256, "Staged provider fixture changed");
+  assert.ok(remote.runner.metadata?.prpTransportModes?.includes("dial_wss"), "Remote daemon must support verified WSS");
+  for (const [group, field] of [["runner", "sha256"], ["provider", "nodeSha256"], ["provider", "wrapperSha256"], ["provider", "fixtureSha256"]]) {
+    assert.equal(remote[group][field], first.baseline.remote[group][field], "Remote provider/daemon artifact changed across matched samples");
+  }
   assert.equal(report.admissionProtocol, "sequential-provider-and-clock-readiness-before-common-epoch", "Use the qualified pre-measurement admission protocol");
   assert.equal(report.setupReadinessPollingMs, 50, "Keep the qualified setup readiness observation interval");
   assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false, "Smoke cannot qualify performance");
@@ -45,6 +83,8 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
   for (const name of names) {
     const result = report.cases.find((item: any) => item.name === name);
     assert.ok(result, `Missing fixed workload ${name}`);
+    const fixed = workloads.find(item => item.name === name)!;
+    for (const field of ["runs", "events", "intervalMs", "prelude", "stop"] as const) assert.equal(result[field], fixed[field], `Fixed workload changed: ${name}.${field}`);
     assert.equal(result.persistedAfterReload, true);
     assert.equal(result.matchedVisibleClocks, result.visible?.length, "Every observed marker needs its original provider clock");
     assert.ok(result.matchedVisibleClocks >= (result.stop ? 3 : result.events), "Missing foreground render/clock observations");
@@ -58,6 +98,55 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
       assert.ok(result.preludeProof?.verifiedAtMs < result.epochMs, "Verify retained prelude before releasing measured emission");
     }
     assert.equal(result.durable.length, result.runs, "Every background run needs independent delta persistence proof");
+    assert.equal(result.runIds?.length, result.runs, "Every remote participant needs its actual run identity");
+    assert.ok(result.runIds.every(uuid), "Missing actual remote run UUID");
+    assert.equal(new Set(result.runIds).size, result.runs, "Duplicate remote run identity");
+    assert.deepEqual(result.durable.map((proof: any) => proof.runId).sort(), [...result.runIds].sort(), "Durable proof run set mismatch");
+    const authority = result.controllerAuthority;
+    assert.equal(authority?.checkedBeforeMeasuredEpoch, true, "Missing remote database controller authority proof");
+    assert.equal(authority?.observedReadOnly, "on", "Remote authority proof must observe a read-only transaction");
+    assert.equal(authority?.observedIsolation, "repeatable read", "Remote authority proof must observe repeatable-read isolation");
+    assert.ok(Number.isSafeInteger(authority?.checkedAtMs) && authority.checkedAtMs > 0
+      && Number.isSafeInteger(authority.connectionClosedAtMs) && authority.connectionClosedAtMs > 0
+      && authority.checkedAtMs <= authority.connectionClosedAtMs && authority.connectionClosedAtMs < result.epochMs,
+    "Close the actual remote authority snapshot before the measured epoch");
+    assert.equal(authority.rows?.length, result.runs, "Every remote run needs its controller checkpoint and lease proof");
+    assert.deepEqual(authority.rows.map((row: any) => row.run_id).sort(), [...result.runIds].sort(), "Remote authority run set mismatch");
+    assert.equal(new Set(authority.rows.map((row: any) => row.issue_id)).size, result.runs, "Duplicate remote issue authority");
+    const authorityByRun = new Map<string, any>();
+    for (const row of authority.rows) {
+      assert.ok(uuid(row.issue_id) && uuid(row.lease_id), "Missing actual remote task/lease UUID");
+      assert.equal(row.company_id, report.companyId, "Remote authority company mismatch");
+      assert.equal(row.environment_id, remote.environmentId, "Remote authority environment mismatch");
+      assert.equal(row.runtime_mode, "native");
+      assert.equal(row.provider, "fake-plugin");
+      assert.match(row.provider_lease_id, /^fake-plugin:\/\//);
+      assert.equal(row.checkpoint_part, "header", "Missing actual database controller checkpoint header");
+      assert.ok(Number.isSafeInteger(row.checkpoint_header_bytes) && row.checkpoint_header_bytes > 0);
+      authorityByRun.set(row.run_id, row);
+    }
+    assert.equal(result.remoteLeases?.length, result.runs, "Every remote run needs post-timing lease cleanup proof");
+    assert.deepEqual(result.remoteLeases.map((lease: any) => lease.runId).sort(), [...result.runIds].sort(), "Remote cleanup run set mismatch");
+    assert.ok(Number.isSafeInteger(result.remoteLeasesCheckedAtMs) && result.remoteLeasesCheckedAtMs >= result.settledAtMs
+      && (!result.stop || result.remoteLeasesCheckedAtMs >= result.stopAcknowledgedAtMs), "Verify remote leases after recorded timing boundaries");
+    assert.equal(result.settledAtMs, result.epochMs + result.drainThroughTerminalMs, "Terminal timing boundary changed");
+    if (result.stop) assert.equal(result.stopAcknowledgedAtMs, result.stopRequestedAtMs + result.stopThroughAcknowledgementMs, "Stop acknowledgement timing boundary changed");
+    for (const lease of result.remoteLeases) {
+      const row = authorityByRun.get(lease.runId);
+      assert.equal(lease.id, row.lease_id); assert.equal(lease.companyId, report.companyId);
+      assert.equal(lease.environmentId, remote.environmentId); assert.equal(lease.provider, row.provider);
+      assert.equal(lease.providerLeaseId, row.provider_lease_id);
+      assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt
+        && lease.cleanupStatus === "success", "Remote lease cleanup remains unsettled");
+      if (lease.terminationReceipt) {
+        const receipt = lease.terminationReceipt;
+        assert.equal(receipt.schema, "paperclip.remote-termination.v1");
+        assert.equal(receipt.companyId, report.companyId); assert.equal(receipt.runId, lease.runId);
+        assert.equal(receipt.leaseId, lease.id); assert.equal(receipt.provider, lease.provider);
+        assert.equal(receipt.providerLeaseId, lease.providerLeaseId);
+        assert.ok(["stopped", "destroyed"].includes(receipt.state), "Invalid recorded remote termination receipt");
+      }
+    }
     for (const proof of result.durable) {
       assert.equal(proof.ordered, true); assert.equal(proof.uniqueSourceIdentities, proof.individualDeltas);
       assert.equal(proof.matchedSourceClocks, proof.individualDeltas, "Every durable measured delta needs an original provider clock");
@@ -91,6 +180,8 @@ const measurements = names.flatMap(name => {
 });
 const result: Record<string, any> = {
   scope: "Compiled-app deterministic-provider/browser acceptance; live-provider qualification remains separate",
+  protocolVersion: first.baseline.protocolVersion, transport: first.baseline.transport,
+  resourceBudgets: first.baseline.resourceBudgets,
   phase, expectation, pairCount: pairs.length, baselineRevision: first.baseline.sourceRevision,
   candidateRevision: first.candidate.sourceRevision, measurementSha256: first.baseline.measurementSha256,
   gatesSha256: createHash("sha256").update(await readFile(new URL("./native-ingestion-performance-gates.mjs", import.meta.url))).digest("hex"),
@@ -102,6 +193,9 @@ if (phase === "compare") {
   assert.ok(arg("noise-report"), "Comparison requires its frozen A/A --noise-report");
   const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8"));
   assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true);
+  assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different remote protocol");
+  assert.equal(noise.transport, result.transport, "Noise report uses a different transport");
+  assert.deepEqual(noise.resourceBudgets, result.resourceBudgets, "Resource budgets changed after A/A calibration");
   assert.equal(noise.baselineRevision, result.baselineRevision); assert.equal(noise.measurementSha256, result.measurementSha256);
   assert.equal(noise.gatesSha256, result.gatesSha256, "Acceptance gates changed after A/A calibration");
   assert.equal(noise.comparatorSha256, result.comparatorSha256, "Comparator changed after A/A calibration");

scripts/native-ingestion-performance-gates.test.mjs

@@ -75,18 +75,41 @@ const workloads = [
   { name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true },
 ];
 function browserReport(revision, latency) {
-  return { protocolVersion: "v2-compact-markers-original-clock-stream", admissionProtocol: "sequential-provider-and-clock-readiness-before-common-epoch", setupReadinessPollingMs: 50, complete: true, cleanup: { complete: true, errors: [] }, qualificationOnly: false, diagnosticOnly: false, performanceAcceptanceEligible: true, browserRecording: { trace: "off", video: "off", screencast: false },
+  const companyId = "00000000-0000-4000-8000-000000000001", environmentId = "00000000-0000-4000-8000-000000000002";
+  const id = (kind, caseIndex, index) => `00000000-0000-4000-8000-${(kind * 100000 + caseIndex * 1000 + index).toString(16).padStart(12, "0")}`;
+  return { protocolVersion: "v3-supported-remote-original-clock-stream", transport: "sandbox-remote", companyId,
+    resourceBudgets: { providerReadinessMs: 300000, fullInvocationMs: 900000, streamingMs: 120000,
+      terminalDrainageMs: 180000, stopAcknowledgementMs: 30000, remoteLeaseCleanupMs: 30000 },
+    remote: { transport: "sandbox-remote", revision, buildRevision: revision, environmentId,
+      runnerPublicUrl: "wss://127.0.0.1:32100",
+      selection: "Normal plugin install + sandbox environment + agent defaultEnvironmentId; no runtime resolver override",
+      plugin: { pluginKey: "paperclip.fake-sandbox-provider" }, tls: { peerVerification: true, caSha256: "e".repeat(64), serverCertificateSha256: "f".repeat(64) },
+      runner: { sha256: "d".repeat(64), metadata: { prpTransportModes: ["dial_wss"] } },
+      provider: { nodeSha256: "b".repeat(64), wrapperSha256: "c".repeat(64), fixtureSha256: "a".repeat(64) } },
+    fixtureSha256: "a".repeat(64), admissionProtocol: "sequential-provider-and-clock-readiness-before-common-epoch", setupReadinessPollingMs: 50, complete: true, cleanup: { complete: true, errors: [] }, qualificationOnly: false, diagnosticOnly: false, performanceAcceptanceEligible: true, browserRecording: { trace: "off", video: "off", screencast: false },
     sourceRevision: revision, compiledBuildRevision: revision, serverArtifact: "compiled", serverEntrySha256: `${revision}-compiled`,
     measurementSha256: "synthetic-fixed-fixture", compiledHotPathHashes: { append: "append", port: "port", runtime: "runtime" },
-    cases: workloads.map(workload => ({ ...workload, stop: Boolean(workload.stop), persistedAfterReload: true,
+    cases: workloads.map((workload, caseIndex) => ({ ...workload, stop: Boolean(workload.stop), persistedAfterReload: true,
       epochMs: 1000, preludeProof: workload.prelude ? { individualDeltas: workload.prelude, uniqueSourceIdentities: workload.prelude, verifiedAtMs: 999 } : undefined,
+      settledAtMs: 1000 + latency, stopRequestedAtMs: workload.stop ? 1000 : undefined,
+      stopAcknowledgedAtMs: workload.stop ? 1000 + latency : undefined, remoteLeasesCheckedAtMs: 1001 + latency,
+      runIds: Array.from({ length: workload.runs }, (_, index) => id(1, caseIndex, index)),
+      controllerAuthority: { checkedBeforeMeasuredEpoch: true, checkedAtMs: 998, connectionClosedAtMs: 999,
+        observedReadOnly: "on", observedIsolation: "repeatable read", rows: Array.from({ length: workload.runs }, (_, index) => ({
+          run_id: id(1, caseIndex, index), issue_id: id(2, caseIndex, index), company_id: companyId,
+          lease_id: id(3, caseIndex, index), environment_id: environmentId, runtime_mode: "native", provider: "fake-plugin",
+          provider_lease_id: `fake-plugin://${id(1, caseIndex, index)}/fixture`, checkpoint_part: "header", checkpoint_header_bytes: 256,
+        })) },
+      remoteLeases: Array.from({ length: workload.runs }, (_, index) => ({ id: id(3, caseIndex, index), runId: id(1, caseIndex, index),
+        companyId, environmentId, provider: "fake-plugin", providerLeaseId: `fake-plugin://${id(1, caseIndex, index)}/fixture`,
+        status: "released", cleanupStatus: "success", releasedAt: "2026-11-03T00:00:00.000Z" })),
       visible: Array(workload.stop ? 3 : workload.events).fill({}), matchedVisibleClocks: workload.stop ? 3 : workload.events,
       firstVisibleAfterEmissionMs: latency, emissionToVisible: { medianMs: latency, p95Ms: latency },
       interChunkGaps: { paintCount: workload.events, count: workload.events - 1, medianMs: latency, maxMs: latency },
       intendedToVisible: { p95Ms: latency }, drainThroughTerminalMs: latency,
       stopThroughTerminalMs: latency, stopThroughAcknowledgementMs: latency,
-      durable: Array.from({ length: workload.runs }, (_, index) => ({ token: String(index), ordered: true,
-        uniqueSourceIdentities: workload.events, individualDeltas: workload.events, matchedSourceClocks: workload.events })),
+      durable: Array.from({ length: workload.runs }, (_, index) => ({ runId: id(1, caseIndex, index), token: String(index), ordered: true,
+        uniqueSourceIdentities: workload.stop ? 3 : workload.events, individualDeltas: workload.stop ? 3 : workload.events, matchedSourceClocks: workload.stop ? 3 : workload.events })),
     })) };
 }
 
@@ -123,7 +146,34 @@ test("browser CLI rejects the three-pairs-above-noise counterexample and ineligi
       ["profile", report => { report.diagnosticOnly = true; }, /Diagnostic profiles/],
       ["profile-ineligible", report => { report.performanceAcceptanceEligible = false; }, /Profile-only evidence/],
       ["missing-retained-prelude", report => { report.cases.find(row => row.prelude).preludeProof.individualDeltas--; }, /stored prelude deltas/],
-      ["legacy-protocol", report => { delete report.protocolVersion; }, /v2 clock protocol/],
+      ["legacy-protocol", report => { delete report.protocolVersion; }, /v3 remote clock protocol/],
+      ["old-local-protocol", report => { report.protocolVersion = "v2-compact-markers-original-clock-stream"; report.transport = "local"; }, /v3 remote clock protocol/],
+      ["wrong-transport", report => { report.transport = "local"; }, /sandbox remote transport/],
+      ["unregistered-resource-budget", report => { report.resourceBudgets.providerReadinessMs = 600000; }, /registered remote setup\/resource budgets/],
+      ["missing-normal-remote", report => { delete report.remote; }, /normal remote runtime provenance/],
+      ["missing-plugin", report => { delete report.remote.plugin; }, /plugin provenance/],
+      ["resolver-override", report => { report.remote.selection = "forced external controller flag"; }, /normal API remote environment selection/],
+      ["tls-unverified", report => { report.remote.tls.peerVerification = false; }, /TLS peer verification/],
+      ["plaintext-remote", report => { report.remote.runnerPublicUrl = "ws://127.0.0.1:32100"; }, /endpoint must use verified WSS/],
+      ["foreign-remote", report => { report.remote.runnerPublicUrl = "wss://example.com:32100"; }, /owned loopback/],
+      ["different-provider-artifact", report => { report.remote.provider.nodeSha256 = "9".repeat(64); }, /artifact changed/],
+      ["fixed-workload-changed", report => { report.cases[3].runs = 10; }, /Fixed workload changed/],
+      ["missing-controller-authority", report => { delete report.cases[3].controllerAuthority; }, /controller authority proof/],
+      ["authority-writable", report => { report.cases[3].controllerAuthority.observedReadOnly = "off"; }, /read-only transaction/],
+      ["authority-wrong-isolation", report => { report.cases[3].controllerAuthority.observedIsolation = "read committed"; }, /repeatable-read isolation/],
+      ["authority-late", report => { report.cases[3].controllerAuthority.checkedAtMs = 1000; }, /before the measured epoch/],
+      ["authority-fractional-closed-time", report => { report.cases[3].controllerAuthority.connectionClosedAtMs = 999.5; }, /before the measured epoch/],
+      ["authority-missing-run", report => { report.cases[3].controllerAuthority.rows.pop(); }, /Every remote run/],
+      ["authority-wrong-run", report => { report.cases[3].controllerAuthority.rows[99].run_id = report.cases[3].controllerAuthority.rows[0].run_id; }, /run set mismatch/],
+      ["authority-wrong-company", report => { report.cases[3].controllerAuthority.rows[99].company_id = "other"; }, /company mismatch/],
+      ["authority-wrong-environment", report => { report.cases[3].controllerAuthority.rows[99].environment_id = "other"; }, /environment mismatch/],
+      ["authority-wrong-provider", report => { report.cases[3].controllerAuthority.rows[99].provider = "fake"; }, /fake-plugin/],
+      ["authority-missing-checkpoint", report => { report.cases[3].controllerAuthority.rows[99].checkpoint_part = null; }, /checkpoint header/],
+      ["missing-lease-cleanup", report => { delete report.cases[3].remoteLeases; }, /lease cleanup proof/],
+      ["lease-pending", report => { report.cases[3].remoteLeases[99].status = "pending_cleanup"; }, /cleanup remains unsettled/],
+      ["lease-wrong-binding", report => { report.cases[3].remoteLeases[99].providerLeaseId = "other"; }, /fake-plugin/],
+      ["lease-check-before-timing", report => { report.cases[3].remoteLeasesCheckedAtMs = report.cases[3].settledAtMs - 1; }, /after recorded timing boundaries/],
+      ["stop-timing-restamped", report => { report.cases[5].stopAcknowledgedAtMs++; }, /timing boundary changed/],
       ["unqualified-admission", report => { delete report.admissionProtocol; }, /admission protocol/],
       ["unqualified-readiness-observation", report => { delete report.setupReadinessPollingMs; }, /readiness observation interval/],
       ["unfinished-cleanup", report => { report.cleanup.complete = false; }, /completed fixture cleanup/],
@@ -138,5 +188,15 @@ test("browser CLI rejects the three-pairs-above-noise counterexample and ineligi
       const rejected = run([`--pairs=${await manifest(name, Array(5).fill(0.30), false, change)}`, `--noise-report=${noise}`, "--require-acceptance"]);
       assert.notEqual(rejected.status, 0); assert.match(rejected.stderr, message);
     }
+    for (const [name, change, message] of [
+      ["old-noise-protocol", report => { report.protocolVersion = "v2-compact-markers-original-clock-stream"; }, /different remote protocol/],
+      ["local-noise-transport", report => { report.transport = "local"; }, /different transport/],
+      ["changed-noise-budget", report => { report.resourceBudgets.providerReadinessMs++; }, /budgets changed after A\/A/],
+    ]) {
+      const changed = JSON.parse(await readFile(noise, "utf8")); change(changed);
+      const file = await save(name + ".json", changed);
+      const rejected = run([`--pairs=${await manifest(name, Array(5).fill(0.30))}`, `--noise-report=${file}`, "--require-acceptance"]);
+      assert.notEqual(rejected.status, 0); assert.match(rejected.stderr, message);
+    }
   } finally { await rm(directory, { recursive: true, force: true }); }
 });

tests/e2e/fixtures/native-ingestion-remote-server.mjs

@@ -0,0 +1,85 @@
+// Test-only assembly of the normal remote sandbox path. The sandbox plugin
+// executes separate OS processes on this same host; TLS and runtime authority
+// follow the production direct-outbound contract.
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { lstat, mkdir, readFile, realpath, writeFile } from 'node:fs/promises';
+import { createConnection } from 'node:net';
+import path from 'node:path';
+import { createServer } from 'node:tls';
+import { pathToFileURL } from 'node:url';
+
+assert.equal(process.env.NODE_ENV, 'test');
+assert.equal(process.env.PAPERCLIP_INGESTION_TRANSPORT, 'sandbox-remote');
+assert.ok(process.env.PAPERCLIP_HOME && process.env.PAPERCLIP_CONFIG);
+assert.ok(!process.env.DATABASE_URL, 'Remote fixture requires its disposable embedded database');
+assert.ok(path.resolve(process.env.PAPERCLIP_CONFIG).startsWith(`${path.resolve(process.env.PAPERCLIP_HOME)}${path.sep}`));
+const ownedHome = await realpath(process.env.PAPERCLIP_HOME);
+const ownedConfiguration = await realpath(process.env.PAPERCLIP_CONFIG);
+assert.ok(ownedConfiguration.startsWith(`${ownedHome}${path.sep}`));
+const configuration = JSON.parse(await readFile(ownedConfiguration, 'utf8'));
+assert.equal(configuration.database.mode, 'embedded-postgres');
+const databaseDirectory = path.resolve(configuration.database.embeddedPostgresDataDir);
+assert.ok((await realpath(path.dirname(databaseDirectory))).startsWith(`${ownedHome}${path.sep}`));
+try {
+  const info = await lstat(databaseDirectory);
+  assert.ok(info.isDirectory() && !info.isSymbolicLink());
+} catch (error) {
+  if (error.code !== 'ENOENT') throw error;
+  await mkdir(databaseDirectory, { mode: 0o700 });
+}
+assert.ok((await realpath(databaseDirectory)).startsWith(`${ownedHome}${path.sep}`));
+assert.ok(process.argv.includes('--compiled-entry=server/dist/index.js'));
+const appRoot = process.cwd();
+const revision = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: appRoot, encoding: 'utf8' }).trim();
+const build = JSON.parse(await readFile(path.join(appRoot, 'server/dist/build-info.json'), 'utf8'));
+assert.equal(build.commit, revision, 'Remote fixture requires the matching compiled application');
+const directory = path.join(process.env.PAPERCLIP_HOME, 'ingestion-remote-fixture');
+await mkdir(directory, { recursive: true, mode: 0o700 });
+const key = path.join(directory, 'loopback-key.pem'), cert = path.join(directory, 'loopback-server.pem');
+const caKey = path.join(directory, 'loopback-ca-key.pem'), ca = path.join(directory, 'loopback-ca.pem');
+execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-sha256', '-days', '2',
+  '-keyout', caKey, '-out', ca, '-subj', '/CN=Paperclip ingestion fixture CA',
+  '-addext', 'basicConstraints=critical,CA:TRUE', '-addext', 'keyUsage=critical,keyCertSign,cRLSign'], { stdio: 'ignore' });
+const request = path.join(directory, 'loopback-server.csr'), extensions = path.join(directory, 'loopback-server.ext');
+await writeFile(extensions, 'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=IP:127.0.0.1,DNS:localhost\n', { mode: 0o600 });
+execFileSync('openssl', ['req', '-new', '-newkey', 'rsa:2048', '-nodes', '-sha256',
+  '-keyout', key, '-out', request, '-subj', '/CN=127.0.0.1'], { stdio: 'ignore' });
+execFileSync('openssl', ['x509', '-req', '-in', request, '-CA', ca, '-CAkey', caKey, '-CAcreateserial',
+  '-out', cert, '-days', '2', '-sha256', '-extfile', extensions], { stdio: 'ignore' });
+execFileSync('openssl', ['verify', '-CAfile', ca, cert], { stdio: 'ignore' });
+const fixture = path.resolve(import.meta.dirname, 'in-feed-codex.mjs');
+const wrapper = path.join(directory, 'codex');
+const quote = value => `'${value.replaceAll("'", "'\\''")}'`;
+await writeFile(wrapper, `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(fixture)} --ingestion-stream-fixture "$@"\n`, { mode: 0o700 });
+const runner = process.env.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH
+  ?? path.join(appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd');
+const runnerMetadata = JSON.parse(execFileSync(runner, ['--build-metadata'], { encoding: 'utf8' }));
+const sockets = new Set();
+const proxy = createServer({ key: await readFile(key), cert: await readFile(cert) }, socket => {
+  const upstream = createConnection({ host: '127.0.0.1', port: Number(process.env.PORT) });
+  for (const connection of [socket, upstream]) {
+    sockets.add(connection); connection.on('close', () => sockets.delete(connection));
+    connection.on('error', () => { socket.destroy(); upstream.destroy(); });
+  }
+  socket.pipe(upstream).pipe(socket);
+});
+await new Promise((resolve, reject) => { proxy.once('error', reject); proxy.listen(0, '127.0.0.1', resolve); });
+const address = proxy.address();
+assert.ok(address && typeof address !== 'string');
+process.env.PAPERCLIP_RUNNER_PUBLIC_URL = `wss://127.0.0.1:${address.port}`;
+process.env.PAPERCLIP_RUNNER_CA_BUNDLE_PATH = ca;
+process.env.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH = runner;
+process.env.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH = wrapper;
+const hash = async file => createHash('sha256').update(await readFile(file)).digest('hex');
+const provenance = { transport: 'sandbox-remote', provisioning: 'First-party fake-plugin; separate OS processes on this same physical host',
+  revision, buildRevision: build.commit, runnerPublicUrl: process.env.PAPERCLIP_RUNNER_PUBLIC_URL,
+  tls: { peerVerification: true, caSha256: await hash(ca), serverCertificateSha256: await hash(cert), subjectAlternativeNames: ['127.0.0.1', 'localhost'] },
+  runner: { path: runner, sha256: await hash(runner), metadata: runnerMetadata },
+  provider: { wrapper, wrapperSha256: await hash(wrapper), node: process.execPath, nodeVersion: process.version, nodeSha256: await hash(process.execPath),
+    fixture, fixtureSha256: await hash(fixture) } };
+await writeFile(path.join(process.env.PAPERCLIP_HOME, 'ingestion-remote-provenance.json'), JSON.stringify(provenance, null, 2) + '\n');
+const { startServer } = await import(pathToFileURL(path.join(appRoot, 'server/dist/index.js')).href);
+try { await startServer(); }
+catch (error) { for (const socket of sockets) socket.destroy(); proxy.close(); throw error; }

tests/e2e/fixtures/native-ingestion-remote.md

@@ -0,0 +1,94 @@
+# Supported remote ingestion fixture
+
+This test assembly selects the production remote execution path through a
+normal plugin installation, sandbox environment and agent `defaultEnvironmentId`.
+The first-party `@paperclipai/plugin-fake-sandbox` plugin executes separate OS
+processes in private temporary directories on the **same physical host**. It
+simulates provisioning; it does not simulate the Runner, PRP connection,
+PostgreSQL append path, WebSocket delivery or browser.
+
+Use a Linux host. Current remote process supervision requires a Linux boot ID
+and process start ticks; matching a Darwin daemon's binary architecture alone
+does not qualify a Darwin remote environment. The fixture does not weaken this
+identity check.
+
+Build and independently install the application normally. The server stamp must
+equal the application's exact source revision. The application must include the
+built fake-sandbox plugin and matching native Runner daemon. The separate H3
+checkout supplies driver tooling and fixture files only.
+
+```sh
+PAPERCLIP_INGESTION_TRANSPORT=sandbox-remote \
+PAPERCLIP_INGESTION_SMOKE=1 \
+PAPERCLIP_INGESTION_APP_ROOT="$APP_ROOT" \
+PAPERCLIP_INGESTION_EVIDENCE_DIR="$NEW_EVIDENCE_DIR" \
+PAPERCLIP_E2E_PORT="$OWNED_FREE_PORT" \
+node node_modules/@playwright/test/cli.js test \
+  --config tests/e2e/playwright-native-ingestion.config.ts
+```
+
+Run this from the test-driver checkout. Smoke emits the original 16-event case
+and exercises Stop separately. It is functional qualification, never performance
+acceptance. Omit the smoke flag only after the complete protocol, application
+revisions, paired-run commands and acceptance rules have been frozen.
+
+The test-only server assembly:
+
+- Generates a private two-day CA and a signed server certificate with
+  `CA:FALSE`, server authentication usage, and `127.0.0.1`/`localhost` subject
+  alternative names, and terminates verified TLS in front of
+  the compiled HTTP server using the existing Runner reverse-proxy contract.
+- Sets the normal `PAPERCLIP_RUNNER_PUBLIC_URL`,
+  `PAPERCLIP_RUNNER_CA_BUNDLE_PATH`, `PAPERCLIP_RUNNER_REMOTE_BINARY_PATH` and
+  `PAPERCLIP_RUNNER_REMOTE_CODEX_PATH` inputs. Artifact staging, version checks,
+  checksum checks, transport authentication and process identity remain owned
+  by production code.
+- Stages a deterministic Codex wrapper with an absolute Node executable and
+  absolute fixture path, because staging relocates the wrapper into the remote
+  runtime. Provenance records the wrapper, Node, provider fixture, daemon and CA
+  hashes. Private keys are not included in provenance.
+- Gives the normal environment an explicit `PATH` containing the recorded Node
+  directory and fixed system directories. The fake provider's default Unix PATH
+  does not contain Homebrew Node; this does not override a command resolver.
+
+Remote mode removes the local Codex command-resolver test hook. It never forces
+`controllerPersistence`, changes local raw journaling or disables certificate
+verification. Before releasing each common emission epoch, a separate raw
+PostgreSQL connection verifies the actual run's company/environment lease and
+database controller checkpoint header in a read-only repeatable-read snapshot.
+The connection reads the actual owned postmaster port, closes before the epoch,
+and neither inserts fixture outcomes nor changes state. Lease verification after
+timed settlement also uses the normal API.
+
+All existing event counts, rates, marker clocks, payload caps, independent delta
+checks, reload, Stop and timing gates remain. Before any Linux remote or paired
+timings, the lead registered **300 seconds** for shared pre-epoch remote setup
+and **900 seconds** for the whole test, allowing normal plugin/daemon staging.
+The local failed protocol retains its original 180/600-second limits. Streaming,
+terminal drainage and Stop acknowledgement keep their original 120/180/30-second
+timeouts. Post-timing lease cleanup has a separate 30-second verification bound.
+Reports record these exact values and the comparator rejects other budgets.
+Remote setup and the complete 100-run workload still require qualification before
+paired measurements; these resource allowances establish no latency success.
+
+Two excluded Mac setup attempts are retained locally:
+
+- `native-ingestion-remote-smoke-ahjnfjv3`: normal plugin/environment installation
+  passed; agent verification failed before a provider turn. The launch shell's
+  plain Node command was independently shown to exit 127 under the fake
+  provider's default PATH. The API exposed `runner_remote_process_launch_failed`.
+- `native-ingestion-remote-smoke-atl7nu5j`: explicit normal environment PATH fixed
+  that missing executable; verification still failed before a provider turn.
+  Static process supervision requires Linux `/proc` fingerprints, so no further
+  Mac attempts or identity bypass were made. The exact second runtime cause is
+  not promoted beyond the retained lifecycle failure.
+
+Both original reports remain failures. Their owned app, TLS and actual
+PostgreSQL ports were independently checked closed, known lease processes were
+absent, and only their recorded temporary homes were removed. No Linux native
+turn has yet passed this fixture at the time of this initial recipe.
+
+The exact certificate-generation block passed a separate local TLS check:
+the matching CA authorized the connection, an untrusted connection failed, and
+an incorrect hostname failed with `ERR_TLS_CERT_ALTNAME_INVALID`. This qualifies
+certificate assembly only. It does not qualify the remote Runner or any timing.

tests/e2e/native-event-ingestion.journey.ts

@@ -1,7 +1,8 @@
 import { createServer, type ServerResponse } from "node:http";
 import { randomUUID, createHash } from "node:crypto";
 import { execFileSync } from "node:child_process";
-import { readFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { readFile, realpath } from "node:fs/promises";
 import path from "node:path";
 import { expect, test } from "@playwright/test";
 import { json } from "./agent-chat.shared";
@@ -15,6 +16,8 @@ type ClockStream = { nonce: string; markerCode: string; opened: boolean; hello:
 type ClockCase = { events: number; intervalMs: number; epochMs?: number; streams: Map<string, ClockStream>; errors: string[] };
 type Case = { name: string; runs: number; events: number; intervalMs: number; prelude: number };
 const smoke = process.env.PAPERCLIP_INGESTION_SMOKE === "1";
+const transport = process.env.PAPERCLIP_INGESTION_TRANSPORT ?? "local";
+const providerReadinessMs = transport === "sandbox-remote" ? 300_000 : 180_000;
 const cases: Case[] = smoke ? [{ name: "qualification", runs: 1, events: 16, intervalMs: 25, prelude: 0 }] : [
   { name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0 },
   { name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0 },
@@ -28,6 +31,57 @@ const stats = (values: number[]) => {
     ...(values.length >= 100 ? { p95Ms: sorted[Math.ceil(sorted.length * 0.95) - 1] } : {}) };
 };
 
+async function remoteControllerAuthority(input: {
+  appRoot: string; companyId: string; environmentId: string; issueIds: string[];
+}) {
+  const configuration = process.env.PAPERCLIP_E2E_SERVER_CONFIG;
+  if (!configuration) throw new Error("Missing owned server configuration");
+  const config = JSON.parse(await readFile(configuration, "utf8"));
+  expect(config.database.mode).toBe("embedded-postgres");
+  const dataDirectory = await realpath(config.database.embeddedPostgresDataDir);
+  const ownedHome = await realpath(path.resolve(path.dirname(configuration), "../.."));
+  expect(dataDirectory.startsWith(`${ownedHome}${path.sep}`)).toBe(true);
+  const owner = (await readFile(path.join(dataDirectory, "postmaster.pid"), "utf8")).split(/\r?\n/);
+  expect(await realpath(owner[1])).toBe(dataDirectory);
+  const port = Number(owner[3]);
+  expect(Number.isSafeInteger(port) && port > 0 && port <= 65535).toBe(true);
+  const require = createRequire(path.join(input.appRoot, "packages/db/package.json"));
+  const postgres = require("postgres");
+  const sql = postgres({ host: "127.0.0.1", port, database: "paperclip", username: "paperclip", password: "paperclip",
+    max: 1, connect_timeout: 10, connection: { application_name: "native_ingestion_remote_authority_proof" } });
+  let proof;
+  try {
+    proof = await sql.begin("isolation level repeatable read read only", async (tx: any) => {
+      const settings = await tx`select current_setting('transaction_read_only') as read_only,
+        current_setting('transaction_isolation') as isolation`;
+      expect(settings[0].read_only).toBe("on");
+      expect(settings[0].isolation).toBe("repeatable read");
+      const rows = await tx`select r.id as run_id, r.company_id, r.native_issue_id as issue_id, r.runtime_mode,
+        l.id as lease_id, l.environment_id, l.provider, l.provider_lease_id,
+        c.part as checkpoint_part, octet_length(c.material::text) as checkpoint_header_bytes
+        from heartbeat_runs r
+        join environment_leases l on l.heartbeat_run_id = r.id and l.company_id = r.company_id
+        left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = r.company_id
+          and c.issue_id = r.native_issue_id and c.part = 'header'
+        where r.company_id = ${input.companyId} and r.native_issue_id = any(${input.issueIds}::uuid[])`;
+      expect(rows).toHaveLength(input.issueIds.length);
+      expect(new Set(rows.map((row: any) => row.issue_id)).size).toBe(input.issueIds.length);
+      for (const row of rows) {
+        expect(row.company_id).toBe(input.companyId);
+        expect(row.runtime_mode).toBe("native");
+        expect(row.environment_id).toBe(input.environmentId);
+        expect(row.provider).toBe("fake-plugin");
+        expect(row.provider_lease_id).toMatch(/^fake-plugin:\/\//);
+        expect(row.checkpoint_part).toBe("header");
+        expect(Number(row.checkpoint_header_bytes)).toBeGreaterThan(0);
+      }
+      return { checkedBeforeMeasuredEpoch: true, checkedAtMs: Date.now(), transaction: "READ ONLY REPEATABLE READ",
+        observedReadOnly: settings[0].read_only, observedIsolation: settings[0].isolation, rows };
+    });
+  } finally { await sql.end({ timeout: 5 }); }
+  return { ...proof, connectionClosedAtMs: Date.now() };
+}
+
 test("native ingestion reaches the shipped UI promptly and survives reload", async ({ page, request }, info) => {
   const pending = new Map<string, Map<string, ServerResponse>>();
   const clocks = new Map<string, ClockCase>();
@@ -147,7 +201,10 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
   const browserRecording = { trace: captureMode(info.project.use.trace), video: captureMode(info.project.use.video),
     screencast: captureMode(info.project.use.trace) !== "off" || captureMode(info.project.use.video) !== "off" };
   const report: Record<string, unknown> = {
-    protocolVersion: "v2-compact-markers-original-clock-stream",
+    protocolVersion: transport === "sandbox-remote" ? "v3-supported-remote-original-clock-stream" : "v2-compact-markers-original-clock-stream",
+    transport,
+    resourceBudgets: { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000,
+      terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 },
     admissionProtocol: "sequential-provider-and-clock-readiness-before-common-epoch",
     setupReadinessPollingMs: 50,
     dependency: "Deterministic Codex app-server fixture; real application/runtime/port/PostgreSQL/live WebSocket/shipped UI",
@@ -177,7 +234,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
   }
   report.fixtureSha256 = createHash("sha256").update(await readFile(path.resolve(import.meta.dirname, "fixtures/in-feed-codex.mjs"))).digest("hex");
   const protocolFiles = ["native-event-ingestion.journey.ts", "playwright-native-ingestion.config.ts", "fixtures/in-feed-codex.mjs",
-    "fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs"];
+    "fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs",
+    "fixtures/native-ingestion-remote-server.mjs"];
   const protocolSources = await Promise.all(protocolFiles.map(async file => ({ file, bytes: await readFile(path.resolve(import.meta.dirname, file)) })));
   report.measurementSha256 = createHash("sha256").update(Buffer.concat(protocolSources.map(source => source.bytes))).digest("hex");
   const harnessRoot = path.resolve(import.meta.dirname, "../..");
@@ -185,7 +243,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
     sourceRevision: execFileSync("git", ["rev-parse", "HEAD"], { cwd: harnessRoot, encoding: "utf8" }).trim(),
     sourceStatus: execFileSync("git", ["status", "--porcelain", "--", "tests/e2e"], { cwd: harnessRoot, encoding: "utf8" }).trim(),
     sourceHashes: Object.fromEntries(protocolSources.map(source => [source.file, createHash("sha256").update(source.bytes).digest("hex")])),
-    driverDependencySource: process.env.PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES ?? "Checkout tooling dependencies",
+    driverDependencySource: process.env.PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES
+      ?? "Independent locked H3 driver install; every measured app owns separate dependencies and compiled output",
     driverLockSha256: createHash("sha256").update(await readFile(path.join(harnessRoot, "pnpm-lock.yaml"))).digest("hex"),
     scope: "Separate browser harness and driver-only tooling; every measured app uses its own frozen clean source and matching compiled stamp",
   };
@@ -193,11 +252,30 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
     await json(await request.patch("/api/instance/settings/experimental", { data: { enableNativeRunner: true, enableWorktreeRunExecution: true } }));
     await json(await request.post(`/api/companies/${company.id}/user-secret-definitions`, { data: { key: "OPENAI_API_KEY", name: "Fixture credential" } }));
     await json(await request.post(`/api/companies/${company.id}/me/user-secrets`, { data: { definitionKey: "OPENAI_API_KEY", value: "not-a-real-model-key" } }));
+    let remoteEnvironment: { id: string } | undefined;
+    if (transport === "sandbox-remote") {
+      const pluginPath = path.join(appRoot, "packages/plugins/paperclip-plugin-fake-sandbox");
+      const plugin = await json(await request.post("/api/plugins/install", { timeout: 120_000,
+        data: { packageName: pluginPath, isLocalPath: true } }));
+      expect(plugin.pluginKey).toBe("paperclip.fake-sandbox-provider");
+      const serverConfig = process.env.PAPERCLIP_E2E_SERVER_CONFIG;
+      if (!serverConfig) throw new Error("Missing owned server configuration for remote fixture provenance");
+      const remoteProvenance = JSON.parse(await readFile(path.resolve(path.dirname(serverConfig), "../..", "ingestion-remote-provenance.json"), "utf8"));
+      remoteEnvironment = await json(await request.post(`/api/companies/${company.id}/environments`, { data: {
+        name: "Native ingestion remote fixture", driver: "sandbox",
+        config: { provider: "fake-plugin", image: "fake:latest", reuseLease: false, runnerLifecycleMode: "per_turn" },
+        envVars: { PATH: `${path.dirname(remoteProvenance.provider.node)}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin` },
+      } }));
+      report.remote = { ...remoteProvenance, plugin: { id: plugin.id, pluginKey: plugin.pluginKey, path: pluginPath },
+        environmentId: remoteEnvironment!.id,
+        selection: "Normal plugin install + sandbox environment + agent defaultEnvironmentId; no runtime resolver override" };
+    }
     const agents = [];
     const agentCount = Math.ceil(Math.max(...cases.map(item => item.runs)) / 50);
     for (let index = 0; index < agentCount; index++) {
       const agent = await json(await request.post(`/api/companies/${company.id}/agents`, { timeout: 120_000, data: {
         name: `Ingestion ${index + 1}`, role: "engineer", adapterType: "paperclip_runner",
+        ...(remoteEnvironment ? { defaultEnvironmentId: remoteEnvironment.id } : {}),
         adapterConfig: { provider: "codex", model: "gpt-5.1-codex-mini", env: {
           OPENAI_API_KEY: { type: "user_secret_ref", key: "OPENAI_API_KEY", version: "latest", required: true },
         } }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true, maxConcurrentRuns: 50 } },
@@ -249,7 +327,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
         issueIds.push(background.id);
         issuedTokens.set(background.id, backgroundToken);
       }
-      const readinessDeadlineMs = Date.now() + 180_000;
+      const readinessDeadlineMs = Date.now() + providerReadinessMs;
       active.phase = "activation-and-provider-readiness";
       for (let index = 1; index < issueIds.length; index++) {
         const id = issueIds[index], token = issuedTokens.get(id)!;
@@ -293,6 +371,10 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
         active.preludeProof = preludeProof;
         expect(count).toBe(workload.prelude); expect(identities.size).toBe(count);
       }
+      const controllerAuthority = remoteEnvironment ? await remoteControllerAuthority({
+        appRoot, companyId: company.id, environmentId: remoteEnvironment.id, issueIds,
+      }) : undefined;
+      if (controllerAuthority) active.controllerAuthority = controllerAuthority;
       const epochMs = Date.now() + 1000;
       state.epochMs = epochMs; active.epochMs = epochMs; active.phase = "streaming";
       const participants = [...pending.get(key)!.keys()];
@@ -324,6 +406,26 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
         (await json(await request.get(`/api/heartbeat-runs/${runs[0].id}`))).resultJson?.nativeCancellation?.dispatchState,
       { timeout: 30_000 }).toBe("acknowledged");
       const acknowledgedAt = stop ? Date.now() : undefined;
+      let remoteLeases: unknown[] | undefined;
+      let remoteLeasesCheckedAtMs: number | undefined;
+      if (remoteEnvironment) {
+        let leases: any[] = [];
+        await expect.poll(async () => {
+          leases = await json(await request.get(`/api/environments/${remoteEnvironment.id}/leases`));
+          return runs.every(run => leases.some(lease => lease.heartbeatRunId === run.id && lease.releasedAt
+            && ["released", "expired", "failed"].includes(lease.status) && lease.cleanupStatus === "success"));
+        }, { timeout: 30_000 }).toBe(true);
+        remoteLeasesCheckedAtMs = Date.now();
+        remoteLeases = runs.map(run => {
+          const lease = leases.find((candidate: any) => candidate.heartbeatRunId === run.id);
+          expect(lease, `Missing normal sandbox lease for ${run.id}`).toBeTruthy();
+          expect(lease.environmentId).toBe(remoteEnvironment.id);
+          expect(lease.providerLeaseId).toMatch(/^fake-plugin:\/\//);
+          return { id: lease.id, runId: run.id, companyId: lease.companyId, environmentId: lease.environmentId, provider: lease.provider,
+            providerLeaseId: lease.providerLeaseId, status: lease.status, releasedAt: lease.releasedAt, cleanupStatus: lease.cleanupStatus,
+            terminationReceipt: lease.metadata?.remoteExecutionTermination };
+        });
+      }
       active.phase = "clock-reconciliation";
       await expect.poll(() => [...state.streams.values()].every(stream => !stream.error && (stop
         ? seenMarkers.every(marker => state.streams.get(nonce)!.records.has(marker.index))
@@ -341,7 +443,9 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
       const { count, medianMs, maxMs } = stats(paintTimes.slice(1).map((at, index) => at - paintTimes[index]));
       const interChunkGaps = { paintCount: paintTimes.length, count, medianMs, maxMs };
       const clockEvidence: Array<Record<string, unknown>> = [];
-      const result = { ...workload, stop, nonce, markerCode, epochMs, preludeProof, clockEvidence, interChunkGaps, runIds: runs.map(run => run.id), participants: participants.length, visible, matchedVisibleClocks: visible.length,
+      const result = { ...workload, stop, nonce, markerCode, epochMs, settledAtMs: settledAt, stopRequestedAtMs: stopRequestedAt,
+        stopAcknowledgedAtMs: acknowledgedAt, preludeProof, controllerAuthority, clockEvidence, interChunkGaps,
+        remoteLeases, remoteLeasesCheckedAtMs, runIds: runs.map(run => run.id), participants: participants.length, visible, matchedVisibleClocks: visible.length,
         firstVisibleAfterEmissionMs: visible[0].visibleMs - visible[0].emittedMs,
         emissionToVisible: stats(visible.map(item => item.visibleMs - item.emittedMs)),
         intendedToVisible: stats(visible.map(item => item.visibleMs - item.intendedMs)),

tests/e2e/playwright-native-ingestion.config.ts

@@ -11,7 +11,15 @@ process.env.PAPERCLIP_INGESTION_BROWSER_RECORDING = browserRecording ? "on" : "o
 const appRoot = path.resolve(process.env.PAPERCLIP_INGESTION_APP_ROOT ?? path.resolve(import.meta.dirname, "../.."));
 const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
 const bootstrap = path.resolve(import.meta.dirname, "fixtures/native-ingestion-bootstrap.mjs");
-const compiledCommand = `node --import ./server/node_modules/tsx/dist/loader.mjs ${quote(bootstrap)} && node --import ./server/node_modules/tsx/dist/loader.mjs server/dist/index.js`;
+const transport = process.env.PAPERCLIP_INGESTION_TRANSPORT ?? "local";
+if (!["local", "sandbox-remote"].includes(transport)) throw new Error("Unknown ingestion transport");
+if (transport === "sandbox-remote" && process.env.DATABASE_URL) {
+  throw new Error("Remote ingestion fixture requires a new disposable embedded database");
+}
+const remoteServer = path.resolve(import.meta.dirname, "fixtures/native-ingestion-remote-server.mjs");
+const compiledEntry = transport === "sandbox-remote"
+  ? `${quote(remoteServer)} --compiled-entry=server/dist/index.js` : "server/dist/index.js";
+const compiledCommand = `node --import ./server/node_modules/tsx/dist/loader.mjs ${quote(bootstrap)} && node --import ./server/node_modules/tsx/dist/loader.mjs ${compiledEntry}`;
 const command = process.env.PAPERCLIP_INGESTION_SERVER_COMMAND ?? compiledCommand;
 const autoStartingOnboard = /\bonboard\b[^;&]*(?:--yes|-y)\b/.test(command);
 if (command.includes("server/dist/index.js") && autoStartingOnboard) {
@@ -26,7 +34,7 @@ const providerHome = fs.mkdtempSync(path.join(os.tmpdir(), "ingestion-provider-"
 export default defineConfig({
   ...base,
   testMatch: "native-event-ingestion.journey.ts",
-  timeout: 600_000,
+  timeout: transport === "sandbox-remote" ? 900_000 : 600_000,
   ...(evidence ? { outputDir: path.join(evidence, "test-results"),
     reporter: [["list"], ["json", { outputFile: path.join(evidence, "playwright.json") }]] as ReporterDescription[] } : {}),
   use: { ...base.use, trace: browserRecording ? "on" : "off", video: "off", actionTimeout: 20_000 },
@@ -37,10 +45,11 @@ export default defineConfig({
     timeout: 240_000,
     env: {
       ...server?.env,
+      PAPERCLIP_INGESTION_TRANSPORT: transport,
       PAPERCLIP_STOP_CODEX_COMMAND: path.resolve(import.meta.dirname, "fixtures/native-ingestion-bin/codex"),
       CODEX_HOME: providerHome,
       OPENAI_API_KEY: "ingestion-fixture-key",
-      NODE_OPTIONS: `${server?.env?.NODE_OPTIONS ?? ""} --import=${path.resolve(import.meta.dirname, "fixtures/composer-stop-codex-loader.mjs")}`,
+      NODE_OPTIONS: `${server?.env?.NODE_OPTIONS ?? ""}${transport === "local" ? ` --import=${path.resolve(import.meta.dirname, "fixtures/composer-stop-codex-loader.mjs")}` : ""}`,
     },
   },
 });