diff --git a/scripts/compare-native-ingestion-browser.ts b/scripts/compare-native-ingestion-browser.ts index 9e87365fe77..469105c2d60 100644 --- a/scripts/compare-native-ingestion-browser.ts +++ b/scripts/compare-native-ingestion-browser.ts @@ -17,12 +17,50 @@ const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ bas assert.ok(manifest.length >= 5, "At least five actual paired application runs are required"); const load = async (file: string) => JSON.parse(await readFile(path.resolve(path.dirname(manifestPath), file), "utf8")); const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await load(entry.baselineFile), candidate: await load(entry.candidateFile) }))); -const names = ["paced", "burst16", "burst128", "burst512", "existing-history", "operator-stop"]; +const workloads = [ + { name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false }, + { name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false }, + { name: "burst128", runs: 10, events: 128, intervalMs: 0, prelude: 0, stop: false }, + { name: "burst512", runs: 100, events: 512, intervalMs: 0, prelude: 0, stop: false }, + { name: "existing-history", runs: 10, events: 128, intervalMs: 0, prelude: 1000, stop: false }, + { name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true }, +]; +const names = workloads.map(item => item.name); const median = (values: number[]) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]!; const first = pairs[0]; +const sha256 = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); +const uuid = (value: unknown) => typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value); for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) { const report = pair[arm]; - assert.equal(report.protocolVersion, "v2-compact-markers-original-clock-stream", "Fresh browser acceptance requires the qualified v2 clock protocol"); + assert.equal(report.protocolVersion, "v3-supported-remote-original-clock-stream", "Fresh browser acceptance requires the qualified v3 remote clock protocol"); + assert.equal(report.transport, "sandbox-remote", "V3 requires the supported sandbox remote transport"); + assert.deepEqual(report.resourceBudgets, { providerReadinessMs: 300_000, fullInvocationMs: 900_000, + streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 }, + "Keep the prospectively registered remote setup/resource budgets and existing streaming/control timeouts"); + const remote = report.remote; + assert.equal(remote?.transport, "sandbox-remote", "Missing normal remote runtime provenance"); + assert.equal(remote?.plugin?.pluginKey, "paperclip.fake-sandbox-provider", "Missing normal remote plugin provenance"); + assert.equal(remote?.selection, "Normal plugin install + sandbox environment + agent defaultEnvironmentId; no runtime resolver override", + "Use normal API remote environment selection"); + assert.ok(uuid(remote.environmentId) && uuid(report.companyId), "Missing actual remote company/environment identity"); + assert.equal(remote.revision, report.sourceRevision); + assert.equal(remote.buildRevision, report.compiledBuildRevision); + assert.equal(remote.tls?.peerVerification, true, "Remote TLS peer verification is required"); + assert.equal(typeof remote.runnerPublicUrl, "string", "Missing actual remote WSS endpoint"); + const endpoint = new URL(remote.runnerPublicUrl); + assert.equal(endpoint.protocol, "wss:", "Actual remote endpoint must use verified WSS"); + assert.equal(endpoint.hostname, "127.0.0.1", "Use the owned loopback remote fixture endpoint"); + assert.ok(Number(endpoint.port) > 0 && !endpoint.username && !endpoint.password && !endpoint.search && !endpoint.hash, + "Use the recorded owned remote fixture endpoint without overrides"); + for (const value of [remote.tls?.caSha256, remote.tls?.serverCertificateSha256, remote.runner?.sha256, + remote.provider?.nodeSha256, remote.provider?.wrapperSha256, remote.provider?.fixtureSha256]) { + assert.ok(sha256(value), "Missing remote TLS, daemon or provider artifact fingerprint"); + } + assert.equal(remote.provider.fixtureSha256, report.fixtureSha256, "Staged provider fixture changed"); + assert.ok(remote.runner.metadata?.prpTransportModes?.includes("dial_wss"), "Remote daemon must support verified WSS"); + for (const [group, field] of [["runner", "sha256"], ["provider", "nodeSha256"], ["provider", "wrapperSha256"], ["provider", "fixtureSha256"]]) { + assert.equal(remote[group][field], first.baseline.remote[group][field], "Remote provider/daemon artifact changed across matched samples"); + } assert.equal(report.admissionProtocol, "sequential-provider-and-clock-readiness-before-common-epoch", "Use the qualified pre-measurement admission protocol"); assert.equal(report.setupReadinessPollingMs, 50, "Keep the qualified setup readiness observation interval"); assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false, "Smoke cannot qualify performance"); @@ -45,6 +83,8 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) for (const name of names) { const result = report.cases.find((item: any) => item.name === name); assert.ok(result, `Missing fixed workload ${name}`); + const fixed = workloads.find(item => item.name === name)!; + for (const field of ["runs", "events", "intervalMs", "prelude", "stop"] as const) assert.equal(result[field], fixed[field], `Fixed workload changed: ${name}.${field}`); assert.equal(result.persistedAfterReload, true); assert.equal(result.matchedVisibleClocks, result.visible?.length, "Every observed marker needs its original provider clock"); assert.ok(result.matchedVisibleClocks >= (result.stop ? 3 : result.events), "Missing foreground render/clock observations"); @@ -58,6 +98,55 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) assert.ok(result.preludeProof?.verifiedAtMs < result.epochMs, "Verify retained prelude before releasing measured emission"); } assert.equal(result.durable.length, result.runs, "Every background run needs independent delta persistence proof"); + assert.equal(result.runIds?.length, result.runs, "Every remote participant needs its actual run identity"); + assert.ok(result.runIds.every(uuid), "Missing actual remote run UUID"); + assert.equal(new Set(result.runIds).size, result.runs, "Duplicate remote run identity"); + assert.deepEqual(result.durable.map((proof: any) => proof.runId).sort(), [...result.runIds].sort(), "Durable proof run set mismatch"); + const authority = result.controllerAuthority; + assert.equal(authority?.checkedBeforeMeasuredEpoch, true, "Missing remote database controller authority proof"); + assert.equal(authority?.observedReadOnly, "on", "Remote authority proof must observe a read-only transaction"); + assert.equal(authority?.observedIsolation, "repeatable read", "Remote authority proof must observe repeatable-read isolation"); + assert.ok(Number.isSafeInteger(authority?.checkedAtMs) && authority.checkedAtMs > 0 + && Number.isSafeInteger(authority.connectionClosedAtMs) && authority.connectionClosedAtMs > 0 + && authority.checkedAtMs <= authority.connectionClosedAtMs && authority.connectionClosedAtMs < result.epochMs, + "Close the actual remote authority snapshot before the measured epoch"); + assert.equal(authority.rows?.length, result.runs, "Every remote run needs its controller checkpoint and lease proof"); + assert.deepEqual(authority.rows.map((row: any) => row.run_id).sort(), [...result.runIds].sort(), "Remote authority run set mismatch"); + assert.equal(new Set(authority.rows.map((row: any) => row.issue_id)).size, result.runs, "Duplicate remote issue authority"); + const authorityByRun = new Map(); + for (const row of authority.rows) { + assert.ok(uuid(row.issue_id) && uuid(row.lease_id), "Missing actual remote task/lease UUID"); + assert.equal(row.company_id, report.companyId, "Remote authority company mismatch"); + assert.equal(row.environment_id, remote.environmentId, "Remote authority environment mismatch"); + assert.equal(row.runtime_mode, "native"); + assert.equal(row.provider, "fake-plugin"); + assert.match(row.provider_lease_id, /^fake-plugin:\/\//); + assert.equal(row.checkpoint_part, "header", "Missing actual database controller checkpoint header"); + assert.ok(Number.isSafeInteger(row.checkpoint_header_bytes) && row.checkpoint_header_bytes > 0); + authorityByRun.set(row.run_id, row); + } + assert.equal(result.remoteLeases?.length, result.runs, "Every remote run needs post-timing lease cleanup proof"); + assert.deepEqual(result.remoteLeases.map((lease: any) => lease.runId).sort(), [...result.runIds].sort(), "Remote cleanup run set mismatch"); + assert.ok(Number.isSafeInteger(result.remoteLeasesCheckedAtMs) && result.remoteLeasesCheckedAtMs >= result.settledAtMs + && (!result.stop || result.remoteLeasesCheckedAtMs >= result.stopAcknowledgedAtMs), "Verify remote leases after recorded timing boundaries"); + assert.equal(result.settledAtMs, result.epochMs + result.drainThroughTerminalMs, "Terminal timing boundary changed"); + if (result.stop) assert.equal(result.stopAcknowledgedAtMs, result.stopRequestedAtMs + result.stopThroughAcknowledgementMs, "Stop acknowledgement timing boundary changed"); + for (const lease of result.remoteLeases) { + const row = authorityByRun.get(lease.runId); + assert.equal(lease.id, row.lease_id); assert.equal(lease.companyId, report.companyId); + assert.equal(lease.environmentId, remote.environmentId); assert.equal(lease.provider, row.provider); + assert.equal(lease.providerLeaseId, row.provider_lease_id); + assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt + && lease.cleanupStatus === "success", "Remote lease cleanup remains unsettled"); + if (lease.terminationReceipt) { + const receipt = lease.terminationReceipt; + assert.equal(receipt.schema, "paperclip.remote-termination.v1"); + assert.equal(receipt.companyId, report.companyId); assert.equal(receipt.runId, lease.runId); + assert.equal(receipt.leaseId, lease.id); assert.equal(receipt.provider, lease.provider); + assert.equal(receipt.providerLeaseId, lease.providerLeaseId); + assert.ok(["stopped", "destroyed"].includes(receipt.state), "Invalid recorded remote termination receipt"); + } + } for (const proof of result.durable) { assert.equal(proof.ordered, true); assert.equal(proof.uniqueSourceIdentities, proof.individualDeltas); assert.equal(proof.matchedSourceClocks, proof.individualDeltas, "Every durable measured delta needs an original provider clock"); @@ -91,6 +180,8 @@ const measurements = names.flatMap(name => { }); const result: Record = { scope: "Compiled-app deterministic-provider/browser acceptance; live-provider qualification remains separate", + protocolVersion: first.baseline.protocolVersion, transport: first.baseline.transport, + resourceBudgets: first.baseline.resourceBudgets, phase, expectation, pairCount: pairs.length, baselineRevision: first.baseline.sourceRevision, candidateRevision: first.candidate.sourceRevision, measurementSha256: first.baseline.measurementSha256, gatesSha256: createHash("sha256").update(await readFile(new URL("./native-ingestion-performance-gates.mjs", import.meta.url))).digest("hex"), @@ -102,6 +193,9 @@ if (phase === "compare") { assert.ok(arg("noise-report"), "Comparison requires its frozen A/A --noise-report"); const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8")); assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true); + assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different remote protocol"); + assert.equal(noise.transport, result.transport, "Noise report uses a different transport"); + assert.deepEqual(noise.resourceBudgets, result.resourceBudgets, "Resource budgets changed after A/A calibration"); assert.equal(noise.baselineRevision, result.baselineRevision); assert.equal(noise.measurementSha256, result.measurementSha256); assert.equal(noise.gatesSha256, result.gatesSha256, "Acceptance gates changed after A/A calibration"); assert.equal(noise.comparatorSha256, result.comparatorSha256, "Comparator changed after A/A calibration"); diff --git a/scripts/native-ingestion-performance-gates.test.mjs b/scripts/native-ingestion-performance-gates.test.mjs index e5aedffb42d..7ca344767bc 100644 --- a/scripts/native-ingestion-performance-gates.test.mjs +++ b/scripts/native-ingestion-performance-gates.test.mjs @@ -75,18 +75,41 @@ const workloads = [ { name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true }, ]; function browserReport(revision, latency) { - return { protocolVersion: "v2-compact-markers-original-clock-stream", admissionProtocol: "sequential-provider-and-clock-readiness-before-common-epoch", setupReadinessPollingMs: 50, complete: true, cleanup: { complete: true, errors: [] }, qualificationOnly: false, diagnosticOnly: false, performanceAcceptanceEligible: true, browserRecording: { trace: "off", video: "off", screencast: false }, + const companyId = "00000000-0000-4000-8000-000000000001", environmentId = "00000000-0000-4000-8000-000000000002"; + const id = (kind, caseIndex, index) => `00000000-0000-4000-8000-${(kind * 100000 + caseIndex * 1000 + index).toString(16).padStart(12, "0")}`; + return { protocolVersion: "v3-supported-remote-original-clock-stream", transport: "sandbox-remote", companyId, + resourceBudgets: { providerReadinessMs: 300000, fullInvocationMs: 900000, streamingMs: 120000, + terminalDrainageMs: 180000, stopAcknowledgementMs: 30000, remoteLeaseCleanupMs: 30000 }, + remote: { transport: "sandbox-remote", revision, buildRevision: revision, environmentId, + runnerPublicUrl: "wss://127.0.0.1:32100", + selection: "Normal plugin install + sandbox environment + agent defaultEnvironmentId; no runtime resolver override", + plugin: { pluginKey: "paperclip.fake-sandbox-provider" }, tls: { peerVerification: true, caSha256: "e".repeat(64), serverCertificateSha256: "f".repeat(64) }, + runner: { sha256: "d".repeat(64), metadata: { prpTransportModes: ["dial_wss"] } }, + provider: { nodeSha256: "b".repeat(64), wrapperSha256: "c".repeat(64), fixtureSha256: "a".repeat(64) } }, + fixtureSha256: "a".repeat(64), admissionProtocol: "sequential-provider-and-clock-readiness-before-common-epoch", setupReadinessPollingMs: 50, complete: true, cleanup: { complete: true, errors: [] }, qualificationOnly: false, diagnosticOnly: false, performanceAcceptanceEligible: true, browserRecording: { trace: "off", video: "off", screencast: false }, sourceRevision: revision, compiledBuildRevision: revision, serverArtifact: "compiled", serverEntrySha256: `${revision}-compiled`, measurementSha256: "synthetic-fixed-fixture", compiledHotPathHashes: { append: "append", port: "port", runtime: "runtime" }, - cases: workloads.map(workload => ({ ...workload, stop: Boolean(workload.stop), persistedAfterReload: true, + cases: workloads.map((workload, caseIndex) => ({ ...workload, stop: Boolean(workload.stop), persistedAfterReload: true, epochMs: 1000, preludeProof: workload.prelude ? { individualDeltas: workload.prelude, uniqueSourceIdentities: workload.prelude, verifiedAtMs: 999 } : undefined, + settledAtMs: 1000 + latency, stopRequestedAtMs: workload.stop ? 1000 : undefined, + stopAcknowledgedAtMs: workload.stop ? 1000 + latency : undefined, remoteLeasesCheckedAtMs: 1001 + latency, + runIds: Array.from({ length: workload.runs }, (_, index) => id(1, caseIndex, index)), + controllerAuthority: { checkedBeforeMeasuredEpoch: true, checkedAtMs: 998, connectionClosedAtMs: 999, + observedReadOnly: "on", observedIsolation: "repeatable read", rows: Array.from({ length: workload.runs }, (_, index) => ({ + run_id: id(1, caseIndex, index), issue_id: id(2, caseIndex, index), company_id: companyId, + lease_id: id(3, caseIndex, index), environment_id: environmentId, runtime_mode: "native", provider: "fake-plugin", + provider_lease_id: `fake-plugin://${id(1, caseIndex, index)}/fixture`, checkpoint_part: "header", checkpoint_header_bytes: 256, + })) }, + remoteLeases: Array.from({ length: workload.runs }, (_, index) => ({ id: id(3, caseIndex, index), runId: id(1, caseIndex, index), + companyId, environmentId, provider: "fake-plugin", providerLeaseId: `fake-plugin://${id(1, caseIndex, index)}/fixture`, + status: "released", cleanupStatus: "success", releasedAt: "2026-11-03T00:00:00.000Z" })), visible: Array(workload.stop ? 3 : workload.events).fill({}), matchedVisibleClocks: workload.stop ? 3 : workload.events, firstVisibleAfterEmissionMs: latency, emissionToVisible: { medianMs: latency, p95Ms: latency }, interChunkGaps: { paintCount: workload.events, count: workload.events - 1, medianMs: latency, maxMs: latency }, intendedToVisible: { p95Ms: latency }, drainThroughTerminalMs: latency, stopThroughTerminalMs: latency, stopThroughAcknowledgementMs: latency, - durable: Array.from({ length: workload.runs }, (_, index) => ({ token: String(index), ordered: true, - uniqueSourceIdentities: workload.events, individualDeltas: workload.events, matchedSourceClocks: workload.events })), + durable: Array.from({ length: workload.runs }, (_, index) => ({ runId: id(1, caseIndex, index), token: String(index), ordered: true, + uniqueSourceIdentities: workload.stop ? 3 : workload.events, individualDeltas: workload.stop ? 3 : workload.events, matchedSourceClocks: workload.stop ? 3 : workload.events })), })) }; } @@ -123,7 +146,34 @@ test("browser CLI rejects the three-pairs-above-noise counterexample and ineligi ["profile", report => { report.diagnosticOnly = true; }, /Diagnostic profiles/], ["profile-ineligible", report => { report.performanceAcceptanceEligible = false; }, /Profile-only evidence/], ["missing-retained-prelude", report => { report.cases.find(row => row.prelude).preludeProof.individualDeltas--; }, /stored prelude deltas/], - ["legacy-protocol", report => { delete report.protocolVersion; }, /v2 clock protocol/], + ["legacy-protocol", report => { delete report.protocolVersion; }, /v3 remote clock protocol/], + ["old-local-protocol", report => { report.protocolVersion = "v2-compact-markers-original-clock-stream"; report.transport = "local"; }, /v3 remote clock protocol/], + ["wrong-transport", report => { report.transport = "local"; }, /sandbox remote transport/], + ["unregistered-resource-budget", report => { report.resourceBudgets.providerReadinessMs = 600000; }, /registered remote setup\/resource budgets/], + ["missing-normal-remote", report => { delete report.remote; }, /normal remote runtime provenance/], + ["missing-plugin", report => { delete report.remote.plugin; }, /plugin provenance/], + ["resolver-override", report => { report.remote.selection = "forced external controller flag"; }, /normal API remote environment selection/], + ["tls-unverified", report => { report.remote.tls.peerVerification = false; }, /TLS peer verification/], + ["plaintext-remote", report => { report.remote.runnerPublicUrl = "ws://127.0.0.1:32100"; }, /endpoint must use verified WSS/], + ["foreign-remote", report => { report.remote.runnerPublicUrl = "wss://example.com:32100"; }, /owned loopback/], + ["different-provider-artifact", report => { report.remote.provider.nodeSha256 = "9".repeat(64); }, /artifact changed/], + ["fixed-workload-changed", report => { report.cases[3].runs = 10; }, /Fixed workload changed/], + ["missing-controller-authority", report => { delete report.cases[3].controllerAuthority; }, /controller authority proof/], + ["authority-writable", report => { report.cases[3].controllerAuthority.observedReadOnly = "off"; }, /read-only transaction/], + ["authority-wrong-isolation", report => { report.cases[3].controllerAuthority.observedIsolation = "read committed"; }, /repeatable-read isolation/], + ["authority-late", report => { report.cases[3].controllerAuthority.checkedAtMs = 1000; }, /before the measured epoch/], + ["authority-fractional-closed-time", report => { report.cases[3].controllerAuthority.connectionClosedAtMs = 999.5; }, /before the measured epoch/], + ["authority-missing-run", report => { report.cases[3].controllerAuthority.rows.pop(); }, /Every remote run/], + ["authority-wrong-run", report => { report.cases[3].controllerAuthority.rows[99].run_id = report.cases[3].controllerAuthority.rows[0].run_id; }, /run set mismatch/], + ["authority-wrong-company", report => { report.cases[3].controllerAuthority.rows[99].company_id = "other"; }, /company mismatch/], + ["authority-wrong-environment", report => { report.cases[3].controllerAuthority.rows[99].environment_id = "other"; }, /environment mismatch/], + ["authority-wrong-provider", report => { report.cases[3].controllerAuthority.rows[99].provider = "fake"; }, /fake-plugin/], + ["authority-missing-checkpoint", report => { report.cases[3].controllerAuthority.rows[99].checkpoint_part = null; }, /checkpoint header/], + ["missing-lease-cleanup", report => { delete report.cases[3].remoteLeases; }, /lease cleanup proof/], + ["lease-pending", report => { report.cases[3].remoteLeases[99].status = "pending_cleanup"; }, /cleanup remains unsettled/], + ["lease-wrong-binding", report => { report.cases[3].remoteLeases[99].providerLeaseId = "other"; }, /fake-plugin/], + ["lease-check-before-timing", report => { report.cases[3].remoteLeasesCheckedAtMs = report.cases[3].settledAtMs - 1; }, /after recorded timing boundaries/], + ["stop-timing-restamped", report => { report.cases[5].stopAcknowledgedAtMs++; }, /timing boundary changed/], ["unqualified-admission", report => { delete report.admissionProtocol; }, /admission protocol/], ["unqualified-readiness-observation", report => { delete report.setupReadinessPollingMs; }, /readiness observation interval/], ["unfinished-cleanup", report => { report.cleanup.complete = false; }, /completed fixture cleanup/], @@ -138,5 +188,15 @@ test("browser CLI rejects the three-pairs-above-noise counterexample and ineligi const rejected = run([`--pairs=${await manifest(name, Array(5).fill(0.30), false, change)}`, `--noise-report=${noise}`, "--require-acceptance"]); assert.notEqual(rejected.status, 0); assert.match(rejected.stderr, message); } + for (const [name, change, message] of [ + ["old-noise-protocol", report => { report.protocolVersion = "v2-compact-markers-original-clock-stream"; }, /different remote protocol/], + ["local-noise-transport", report => { report.transport = "local"; }, /different transport/], + ["changed-noise-budget", report => { report.resourceBudgets.providerReadinessMs++; }, /budgets changed after A\/A/], + ]) { + const changed = JSON.parse(await readFile(noise, "utf8")); change(changed); + const file = await save(name + ".json", changed); + const rejected = run([`--pairs=${await manifest(name, Array(5).fill(0.30))}`, `--noise-report=${file}`, "--require-acceptance"]); + assert.notEqual(rejected.status, 0); assert.match(rejected.stderr, message); + } } finally { await rm(directory, { recursive: true, force: true }); } }); diff --git a/tests/e2e/fixtures/native-ingestion-remote-server.mjs b/tests/e2e/fixtures/native-ingestion-remote-server.mjs new file mode 100644 index 00000000000..55ff4d79d47 --- /dev/null +++ b/tests/e2e/fixtures/native-ingestion-remote-server.mjs @@ -0,0 +1,85 @@ +// Test-only assembly of the normal remote sandbox path. The sandbox plugin +// executes separate OS processes on this same host; TLS and runtime authority +// follow the production direct-outbound contract. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { lstat, mkdir, readFile, realpath, writeFile } from 'node:fs/promises'; +import { createConnection } from 'node:net'; +import path from 'node:path'; +import { createServer } from 'node:tls'; +import { pathToFileURL } from 'node:url'; + +assert.equal(process.env.NODE_ENV, 'test'); +assert.equal(process.env.PAPERCLIP_INGESTION_TRANSPORT, 'sandbox-remote'); +assert.ok(process.env.PAPERCLIP_HOME && process.env.PAPERCLIP_CONFIG); +assert.ok(!process.env.DATABASE_URL, 'Remote fixture requires its disposable embedded database'); +assert.ok(path.resolve(process.env.PAPERCLIP_CONFIG).startsWith(`${path.resolve(process.env.PAPERCLIP_HOME)}${path.sep}`)); +const ownedHome = await realpath(process.env.PAPERCLIP_HOME); +const ownedConfiguration = await realpath(process.env.PAPERCLIP_CONFIG); +assert.ok(ownedConfiguration.startsWith(`${ownedHome}${path.sep}`)); +const configuration = JSON.parse(await readFile(ownedConfiguration, 'utf8')); +assert.equal(configuration.database.mode, 'embedded-postgres'); +const databaseDirectory = path.resolve(configuration.database.embeddedPostgresDataDir); +assert.ok((await realpath(path.dirname(databaseDirectory))).startsWith(`${ownedHome}${path.sep}`)); +try { + const info = await lstat(databaseDirectory); + assert.ok(info.isDirectory() && !info.isSymbolicLink()); +} catch (error) { + if (error.code !== 'ENOENT') throw error; + await mkdir(databaseDirectory, { mode: 0o700 }); +} +assert.ok((await realpath(databaseDirectory)).startsWith(`${ownedHome}${path.sep}`)); +assert.ok(process.argv.includes('--compiled-entry=server/dist/index.js')); +const appRoot = process.cwd(); +const revision = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: appRoot, encoding: 'utf8' }).trim(); +const build = JSON.parse(await readFile(path.join(appRoot, 'server/dist/build-info.json'), 'utf8')); +assert.equal(build.commit, revision, 'Remote fixture requires the matching compiled application'); +const directory = path.join(process.env.PAPERCLIP_HOME, 'ingestion-remote-fixture'); +await mkdir(directory, { recursive: true, mode: 0o700 }); +const key = path.join(directory, 'loopback-key.pem'), cert = path.join(directory, 'loopback-server.pem'); +const caKey = path.join(directory, 'loopback-ca-key.pem'), ca = path.join(directory, 'loopback-ca.pem'); +execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-sha256', '-days', '2', + '-keyout', caKey, '-out', ca, '-subj', '/CN=Paperclip ingestion fixture CA', + '-addext', 'basicConstraints=critical,CA:TRUE', '-addext', 'keyUsage=critical,keyCertSign,cRLSign'], { stdio: 'ignore' }); +const request = path.join(directory, 'loopback-server.csr'), extensions = path.join(directory, 'loopback-server.ext'); +await writeFile(extensions, 'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=IP:127.0.0.1,DNS:localhost\n', { mode: 0o600 }); +execFileSync('openssl', ['req', '-new', '-newkey', 'rsa:2048', '-nodes', '-sha256', + '-keyout', key, '-out', request, '-subj', '/CN=127.0.0.1'], { stdio: 'ignore' }); +execFileSync('openssl', ['x509', '-req', '-in', request, '-CA', ca, '-CAkey', caKey, '-CAcreateserial', + '-out', cert, '-days', '2', '-sha256', '-extfile', extensions], { stdio: 'ignore' }); +execFileSync('openssl', ['verify', '-CAfile', ca, cert], { stdio: 'ignore' }); +const fixture = path.resolve(import.meta.dirname, 'in-feed-codex.mjs'); +const wrapper = path.join(directory, 'codex'); +const quote = value => `'${value.replaceAll("'", "'\\''")}'`; +await writeFile(wrapper, `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(fixture)} --ingestion-stream-fixture "$@"\n`, { mode: 0o700 }); +const runner = process.env.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH + ?? path.join(appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd'); +const runnerMetadata = JSON.parse(execFileSync(runner, ['--build-metadata'], { encoding: 'utf8' })); +const sockets = new Set(); +const proxy = createServer({ key: await readFile(key), cert: await readFile(cert) }, socket => { + const upstream = createConnection({ host: '127.0.0.1', port: Number(process.env.PORT) }); + for (const connection of [socket, upstream]) { + sockets.add(connection); connection.on('close', () => sockets.delete(connection)); + connection.on('error', () => { socket.destroy(); upstream.destroy(); }); + } + socket.pipe(upstream).pipe(socket); +}); +await new Promise((resolve, reject) => { proxy.once('error', reject); proxy.listen(0, '127.0.0.1', resolve); }); +const address = proxy.address(); +assert.ok(address && typeof address !== 'string'); +process.env.PAPERCLIP_RUNNER_PUBLIC_URL = `wss://127.0.0.1:${address.port}`; +process.env.PAPERCLIP_RUNNER_CA_BUNDLE_PATH = ca; +process.env.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH = runner; +process.env.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH = wrapper; +const hash = async file => createHash('sha256').update(await readFile(file)).digest('hex'); +const provenance = { transport: 'sandbox-remote', provisioning: 'First-party fake-plugin; separate OS processes on this same physical host', + revision, buildRevision: build.commit, runnerPublicUrl: process.env.PAPERCLIP_RUNNER_PUBLIC_URL, + tls: { peerVerification: true, caSha256: await hash(ca), serverCertificateSha256: await hash(cert), subjectAlternativeNames: ['127.0.0.1', 'localhost'] }, + runner: { path: runner, sha256: await hash(runner), metadata: runnerMetadata }, + provider: { wrapper, wrapperSha256: await hash(wrapper), node: process.execPath, nodeVersion: process.version, nodeSha256: await hash(process.execPath), + fixture, fixtureSha256: await hash(fixture) } }; +await writeFile(path.join(process.env.PAPERCLIP_HOME, 'ingestion-remote-provenance.json'), JSON.stringify(provenance, null, 2) + '\n'); +const { startServer } = await import(pathToFileURL(path.join(appRoot, 'server/dist/index.js')).href); +try { await startServer(); } +catch (error) { for (const socket of sockets) socket.destroy(); proxy.close(); throw error; } diff --git a/tests/e2e/fixtures/native-ingestion-remote.md b/tests/e2e/fixtures/native-ingestion-remote.md new file mode 100644 index 00000000000..94c0d150787 --- /dev/null +++ b/tests/e2e/fixtures/native-ingestion-remote.md @@ -0,0 +1,94 @@ +# Supported remote ingestion fixture + +This test assembly selects the production remote execution path through a +normal plugin installation, sandbox environment and agent `defaultEnvironmentId`. +The first-party `@paperclipai/plugin-fake-sandbox` plugin executes separate OS +processes in private temporary directories on the **same physical host**. It +simulates provisioning; it does not simulate the Runner, PRP connection, +PostgreSQL append path, WebSocket delivery or browser. + +Use a Linux host. Current remote process supervision requires a Linux boot ID +and process start ticks; matching a Darwin daemon's binary architecture alone +does not qualify a Darwin remote environment. The fixture does not weaken this +identity check. + +Build and independently install the application normally. The server stamp must +equal the application's exact source revision. The application must include the +built fake-sandbox plugin and matching native Runner daemon. The separate H3 +checkout supplies driver tooling and fixture files only. + +```sh +PAPERCLIP_INGESTION_TRANSPORT=sandbox-remote \ +PAPERCLIP_INGESTION_SMOKE=1 \ +PAPERCLIP_INGESTION_APP_ROOT="$APP_ROOT" \ +PAPERCLIP_INGESTION_EVIDENCE_DIR="$NEW_EVIDENCE_DIR" \ +PAPERCLIP_E2E_PORT="$OWNED_FREE_PORT" \ +node node_modules/@playwright/test/cli.js test \ + --config tests/e2e/playwright-native-ingestion.config.ts +``` + +Run this from the test-driver checkout. Smoke emits the original 16-event case +and exercises Stop separately. It is functional qualification, never performance +acceptance. Omit the smoke flag only after the complete protocol, application +revisions, paired-run commands and acceptance rules have been frozen. + +The test-only server assembly: + +- Generates a private two-day CA and a signed server certificate with + `CA:FALSE`, server authentication usage, and `127.0.0.1`/`localhost` subject + alternative names, and terminates verified TLS in front of + the compiled HTTP server using the existing Runner reverse-proxy contract. +- Sets the normal `PAPERCLIP_RUNNER_PUBLIC_URL`, + `PAPERCLIP_RUNNER_CA_BUNDLE_PATH`, `PAPERCLIP_RUNNER_REMOTE_BINARY_PATH` and + `PAPERCLIP_RUNNER_REMOTE_CODEX_PATH` inputs. Artifact staging, version checks, + checksum checks, transport authentication and process identity remain owned + by production code. +- Stages a deterministic Codex wrapper with an absolute Node executable and + absolute fixture path, because staging relocates the wrapper into the remote + runtime. Provenance records the wrapper, Node, provider fixture, daemon and CA + hashes. Private keys are not included in provenance. +- Gives the normal environment an explicit `PATH` containing the recorded Node + directory and fixed system directories. The fake provider's default Unix PATH + does not contain Homebrew Node; this does not override a command resolver. + +Remote mode removes the local Codex command-resolver test hook. It never forces +`controllerPersistence`, changes local raw journaling or disables certificate +verification. Before releasing each common emission epoch, a separate raw +PostgreSQL connection verifies the actual run's company/environment lease and +database controller checkpoint header in a read-only repeatable-read snapshot. +The connection reads the actual owned postmaster port, closes before the epoch, +and neither inserts fixture outcomes nor changes state. Lease verification after +timed settlement also uses the normal API. + +All existing event counts, rates, marker clocks, payload caps, independent delta +checks, reload, Stop and timing gates remain. Before any Linux remote or paired +timings, the lead registered **300 seconds** for shared pre-epoch remote setup +and **900 seconds** for the whole test, allowing normal plugin/daemon staging. +The local failed protocol retains its original 180/600-second limits. Streaming, +terminal drainage and Stop acknowledgement keep their original 120/180/30-second +timeouts. Post-timing lease cleanup has a separate 30-second verification bound. +Reports record these exact values and the comparator rejects other budgets. +Remote setup and the complete 100-run workload still require qualification before +paired measurements; these resource allowances establish no latency success. + +Two excluded Mac setup attempts are retained locally: + +- `native-ingestion-remote-smoke-ahjnfjv3`: normal plugin/environment installation + passed; agent verification failed before a provider turn. The launch shell's + plain Node command was independently shown to exit 127 under the fake + provider's default PATH. The API exposed `runner_remote_process_launch_failed`. +- `native-ingestion-remote-smoke-atl7nu5j`: explicit normal environment PATH fixed + that missing executable; verification still failed before a provider turn. + Static process supervision requires Linux `/proc` fingerprints, so no further + Mac attempts or identity bypass were made. The exact second runtime cause is + not promoted beyond the retained lifecycle failure. + +Both original reports remain failures. Their owned app, TLS and actual +PostgreSQL ports were independently checked closed, known lease processes were +absent, and only their recorded temporary homes were removed. No Linux native +turn has yet passed this fixture at the time of this initial recipe. + +The exact certificate-generation block passed a separate local TLS check: +the matching CA authorized the connection, an untrusted connection failed, and +an incorrect hostname failed with `ERR_TLS_CERT_ALTNAME_INVALID`. This qualifies +certificate assembly only. It does not qualify the remote Runner or any timing. diff --git a/tests/e2e/native-event-ingestion.journey.ts b/tests/e2e/native-event-ingestion.journey.ts index cb1ca8960ef..15bd0631b01 100644 --- a/tests/e2e/native-event-ingestion.journey.ts +++ b/tests/e2e/native-event-ingestion.journey.ts @@ -1,7 +1,8 @@ import { createServer, type ServerResponse } from "node:http"; import { randomUUID, createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; -import { readFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { readFile, realpath } from "node:fs/promises"; import path from "node:path"; import { expect, test } from "@playwright/test"; import { json } from "./agent-chat.shared"; @@ -15,6 +16,8 @@ type ClockStream = { nonce: string; markerCode: string; opened: boolean; hello: type ClockCase = { events: number; intervalMs: number; epochMs?: number; streams: Map; errors: string[] }; type Case = { name: string; runs: number; events: number; intervalMs: number; prelude: number }; const smoke = process.env.PAPERCLIP_INGESTION_SMOKE === "1"; +const transport = process.env.PAPERCLIP_INGESTION_TRANSPORT ?? "local"; +const providerReadinessMs = transport === "sandbox-remote" ? 300_000 : 180_000; const cases: Case[] = smoke ? [{ name: "qualification", runs: 1, events: 16, intervalMs: 25, prelude: 0 }] : [ { name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0 }, { name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0 }, @@ -28,6 +31,57 @@ const stats = (values: number[]) => { ...(values.length >= 100 ? { p95Ms: sorted[Math.ceil(sorted.length * 0.95) - 1] } : {}) }; }; +async function remoteControllerAuthority(input: { + appRoot: string; companyId: string; environmentId: string; issueIds: string[]; +}) { + const configuration = process.env.PAPERCLIP_E2E_SERVER_CONFIG; + if (!configuration) throw new Error("Missing owned server configuration"); + const config = JSON.parse(await readFile(configuration, "utf8")); + expect(config.database.mode).toBe("embedded-postgres"); + const dataDirectory = await realpath(config.database.embeddedPostgresDataDir); + const ownedHome = await realpath(path.resolve(path.dirname(configuration), "../..")); + expect(dataDirectory.startsWith(`${ownedHome}${path.sep}`)).toBe(true); + const owner = (await readFile(path.join(dataDirectory, "postmaster.pid"), "utf8")).split(/\r?\n/); + expect(await realpath(owner[1])).toBe(dataDirectory); + const port = Number(owner[3]); + expect(Number.isSafeInteger(port) && port > 0 && port <= 65535).toBe(true); + const require = createRequire(path.join(input.appRoot, "packages/db/package.json")); + const postgres = require("postgres"); + const sql = postgres({ host: "127.0.0.1", port, database: "paperclip", username: "paperclip", password: "paperclip", + max: 1, connect_timeout: 10, connection: { application_name: "native_ingestion_remote_authority_proof" } }); + let proof; + try { + proof = await sql.begin("isolation level repeatable read read only", async (tx: any) => { + const settings = await tx`select current_setting('transaction_read_only') as read_only, + current_setting('transaction_isolation') as isolation`; + expect(settings[0].read_only).toBe("on"); + expect(settings[0].isolation).toBe("repeatable read"); + const rows = await tx`select r.id as run_id, r.company_id, r.native_issue_id as issue_id, r.runtime_mode, + l.id as lease_id, l.environment_id, l.provider, l.provider_lease_id, + c.part as checkpoint_part, octet_length(c.material::text) as checkpoint_header_bytes + from heartbeat_runs r + join environment_leases l on l.heartbeat_run_id = r.id and l.company_id = r.company_id + left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = r.company_id + and c.issue_id = r.native_issue_id and c.part = 'header' + where r.company_id = ${input.companyId} and r.native_issue_id = any(${input.issueIds}::uuid[])`; + expect(rows).toHaveLength(input.issueIds.length); + expect(new Set(rows.map((row: any) => row.issue_id)).size).toBe(input.issueIds.length); + for (const row of rows) { + expect(row.company_id).toBe(input.companyId); + expect(row.runtime_mode).toBe("native"); + expect(row.environment_id).toBe(input.environmentId); + expect(row.provider).toBe("fake-plugin"); + expect(row.provider_lease_id).toMatch(/^fake-plugin:\/\//); + expect(row.checkpoint_part).toBe("header"); + expect(Number(row.checkpoint_header_bytes)).toBeGreaterThan(0); + } + return { checkedBeforeMeasuredEpoch: true, checkedAtMs: Date.now(), transaction: "READ ONLY REPEATABLE READ", + observedReadOnly: settings[0].read_only, observedIsolation: settings[0].isolation, rows }; + }); + } finally { await sql.end({ timeout: 5 }); } + return { ...proof, connectionClosedAtMs: Date.now() }; +} + test("native ingestion reaches the shipped UI promptly and survives reload", async ({ page, request }, info) => { const pending = new Map>(); const clocks = new Map(); @@ -147,7 +201,10 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const browserRecording = { trace: captureMode(info.project.use.trace), video: captureMode(info.project.use.video), screencast: captureMode(info.project.use.trace) !== "off" || captureMode(info.project.use.video) !== "off" }; const report: Record = { - protocolVersion: "v2-compact-markers-original-clock-stream", + protocolVersion: transport === "sandbox-remote" ? "v3-supported-remote-original-clock-stream" : "v2-compact-markers-original-clock-stream", + transport, + resourceBudgets: { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000, + terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 }, admissionProtocol: "sequential-provider-and-clock-readiness-before-common-epoch", setupReadinessPollingMs: 50, dependency: "Deterministic Codex app-server fixture; real application/runtime/port/PostgreSQL/live WebSocket/shipped UI", @@ -177,7 +234,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy } report.fixtureSha256 = createHash("sha256").update(await readFile(path.resolve(import.meta.dirname, "fixtures/in-feed-codex.mjs"))).digest("hex"); const protocolFiles = ["native-event-ingestion.journey.ts", "playwright-native-ingestion.config.ts", "fixtures/in-feed-codex.mjs", - "fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs"]; + "fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs", + "fixtures/native-ingestion-remote-server.mjs"]; const protocolSources = await Promise.all(protocolFiles.map(async file => ({ file, bytes: await readFile(path.resolve(import.meta.dirname, file)) }))); report.measurementSha256 = createHash("sha256").update(Buffer.concat(protocolSources.map(source => source.bytes))).digest("hex"); const harnessRoot = path.resolve(import.meta.dirname, "../.."); @@ -185,7 +243,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy sourceRevision: execFileSync("git", ["rev-parse", "HEAD"], { cwd: harnessRoot, encoding: "utf8" }).trim(), sourceStatus: execFileSync("git", ["status", "--porcelain", "--", "tests/e2e"], { cwd: harnessRoot, encoding: "utf8" }).trim(), sourceHashes: Object.fromEntries(protocolSources.map(source => [source.file, createHash("sha256").update(source.bytes).digest("hex")])), - driverDependencySource: process.env.PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES ?? "Checkout tooling dependencies", + driverDependencySource: process.env.PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES + ?? "Independent locked H3 driver install; every measured app owns separate dependencies and compiled output", driverLockSha256: createHash("sha256").update(await readFile(path.join(harnessRoot, "pnpm-lock.yaml"))).digest("hex"), scope: "Separate browser harness and driver-only tooling; every measured app uses its own frozen clean source and matching compiled stamp", }; @@ -193,11 +252,30 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy await json(await request.patch("/api/instance/settings/experimental", { data: { enableNativeRunner: true, enableWorktreeRunExecution: true } })); await json(await request.post(`/api/companies/${company.id}/user-secret-definitions`, { data: { key: "OPENAI_API_KEY", name: "Fixture credential" } })); await json(await request.post(`/api/companies/${company.id}/me/user-secrets`, { data: { definitionKey: "OPENAI_API_KEY", value: "not-a-real-model-key" } })); + let remoteEnvironment: { id: string } | undefined; + if (transport === "sandbox-remote") { + const pluginPath = path.join(appRoot, "packages/plugins/paperclip-plugin-fake-sandbox"); + const plugin = await json(await request.post("/api/plugins/install", { timeout: 120_000, + data: { packageName: pluginPath, isLocalPath: true } })); + expect(plugin.pluginKey).toBe("paperclip.fake-sandbox-provider"); + const serverConfig = process.env.PAPERCLIP_E2E_SERVER_CONFIG; + if (!serverConfig) throw new Error("Missing owned server configuration for remote fixture provenance"); + const remoteProvenance = JSON.parse(await readFile(path.resolve(path.dirname(serverConfig), "../..", "ingestion-remote-provenance.json"), "utf8")); + remoteEnvironment = await json(await request.post(`/api/companies/${company.id}/environments`, { data: { + name: "Native ingestion remote fixture", driver: "sandbox", + config: { provider: "fake-plugin", image: "fake:latest", reuseLease: false, runnerLifecycleMode: "per_turn" }, + envVars: { PATH: `${path.dirname(remoteProvenance.provider.node)}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin` }, + } })); + report.remote = { ...remoteProvenance, plugin: { id: plugin.id, pluginKey: plugin.pluginKey, path: pluginPath }, + environmentId: remoteEnvironment!.id, + selection: "Normal plugin install + sandbox environment + agent defaultEnvironmentId; no runtime resolver override" }; + } const agents = []; const agentCount = Math.ceil(Math.max(...cases.map(item => item.runs)) / 50); for (let index = 0; index < agentCount; index++) { const agent = await json(await request.post(`/api/companies/${company.id}/agents`, { timeout: 120_000, data: { name: `Ingestion ${index + 1}`, role: "engineer", adapterType: "paperclip_runner", + ...(remoteEnvironment ? { defaultEnvironmentId: remoteEnvironment.id } : {}), adapterConfig: { provider: "codex", model: "gpt-5.1-codex-mini", env: { OPENAI_API_KEY: { type: "user_secret_ref", key: "OPENAI_API_KEY", version: "latest", required: true }, } }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true, maxConcurrentRuns: 50 } }, @@ -249,7 +327,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy issueIds.push(background.id); issuedTokens.set(background.id, backgroundToken); } - const readinessDeadlineMs = Date.now() + 180_000; + const readinessDeadlineMs = Date.now() + providerReadinessMs; active.phase = "activation-and-provider-readiness"; for (let index = 1; index < issueIds.length; index++) { const id = issueIds[index], token = issuedTokens.get(id)!; @@ -293,6 +371,10 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy active.preludeProof = preludeProof; expect(count).toBe(workload.prelude); expect(identities.size).toBe(count); } + const controllerAuthority = remoteEnvironment ? await remoteControllerAuthority({ + appRoot, companyId: company.id, environmentId: remoteEnvironment.id, issueIds, + }) : undefined; + if (controllerAuthority) active.controllerAuthority = controllerAuthority; const epochMs = Date.now() + 1000; state.epochMs = epochMs; active.epochMs = epochMs; active.phase = "streaming"; const participants = [...pending.get(key)!.keys()]; @@ -324,6 +406,26 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy (await json(await request.get(`/api/heartbeat-runs/${runs[0].id}`))).resultJson?.nativeCancellation?.dispatchState, { timeout: 30_000 }).toBe("acknowledged"); const acknowledgedAt = stop ? Date.now() : undefined; + let remoteLeases: unknown[] | undefined; + let remoteLeasesCheckedAtMs: number | undefined; + if (remoteEnvironment) { + let leases: any[] = []; + await expect.poll(async () => { + leases = await json(await request.get(`/api/environments/${remoteEnvironment.id}/leases`)); + return runs.every(run => leases.some(lease => lease.heartbeatRunId === run.id && lease.releasedAt + && ["released", "expired", "failed"].includes(lease.status) && lease.cleanupStatus === "success")); + }, { timeout: 30_000 }).toBe(true); + remoteLeasesCheckedAtMs = Date.now(); + remoteLeases = runs.map(run => { + const lease = leases.find((candidate: any) => candidate.heartbeatRunId === run.id); + expect(lease, `Missing normal sandbox lease for ${run.id}`).toBeTruthy(); + expect(lease.environmentId).toBe(remoteEnvironment.id); + expect(lease.providerLeaseId).toMatch(/^fake-plugin:\/\//); + return { id: lease.id, runId: run.id, companyId: lease.companyId, environmentId: lease.environmentId, provider: lease.provider, + providerLeaseId: lease.providerLeaseId, status: lease.status, releasedAt: lease.releasedAt, cleanupStatus: lease.cleanupStatus, + terminationReceipt: lease.metadata?.remoteExecutionTermination }; + }); + } active.phase = "clock-reconciliation"; await expect.poll(() => [...state.streams.values()].every(stream => !stream.error && (stop ? seenMarkers.every(marker => state.streams.get(nonce)!.records.has(marker.index)) @@ -341,7 +443,9 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const { count, medianMs, maxMs } = stats(paintTimes.slice(1).map((at, index) => at - paintTimes[index])); const interChunkGaps = { paintCount: paintTimes.length, count, medianMs, maxMs }; const clockEvidence: Array> = []; - const result = { ...workload, stop, nonce, markerCode, epochMs, preludeProof, clockEvidence, interChunkGaps, runIds: runs.map(run => run.id), participants: participants.length, visible, matchedVisibleClocks: visible.length, + const result = { ...workload, stop, nonce, markerCode, epochMs, settledAtMs: settledAt, stopRequestedAtMs: stopRequestedAt, + stopAcknowledgedAtMs: acknowledgedAt, preludeProof, controllerAuthority, clockEvidence, interChunkGaps, + remoteLeases, remoteLeasesCheckedAtMs, runIds: runs.map(run => run.id), participants: participants.length, visible, matchedVisibleClocks: visible.length, firstVisibleAfterEmissionMs: visible[0].visibleMs - visible[0].emittedMs, emissionToVisible: stats(visible.map(item => item.visibleMs - item.emittedMs)), intendedToVisible: stats(visible.map(item => item.visibleMs - item.intendedMs)), diff --git a/tests/e2e/playwright-native-ingestion.config.ts b/tests/e2e/playwright-native-ingestion.config.ts index 84ecdf4bff9..60893d36650 100644 --- a/tests/e2e/playwright-native-ingestion.config.ts +++ b/tests/e2e/playwright-native-ingestion.config.ts @@ -11,7 +11,15 @@ process.env.PAPERCLIP_INGESTION_BROWSER_RECORDING = browserRecording ? "on" : "o const appRoot = path.resolve(process.env.PAPERCLIP_INGESTION_APP_ROOT ?? path.resolve(import.meta.dirname, "../..")); const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; const bootstrap = path.resolve(import.meta.dirname, "fixtures/native-ingestion-bootstrap.mjs"); -const compiledCommand = `node --import ./server/node_modules/tsx/dist/loader.mjs ${quote(bootstrap)} && node --import ./server/node_modules/tsx/dist/loader.mjs server/dist/index.js`; +const transport = process.env.PAPERCLIP_INGESTION_TRANSPORT ?? "local"; +if (!["local", "sandbox-remote"].includes(transport)) throw new Error("Unknown ingestion transport"); +if (transport === "sandbox-remote" && process.env.DATABASE_URL) { + throw new Error("Remote ingestion fixture requires a new disposable embedded database"); +} +const remoteServer = path.resolve(import.meta.dirname, "fixtures/native-ingestion-remote-server.mjs"); +const compiledEntry = transport === "sandbox-remote" + ? `${quote(remoteServer)} --compiled-entry=server/dist/index.js` : "server/dist/index.js"; +const compiledCommand = `node --import ./server/node_modules/tsx/dist/loader.mjs ${quote(bootstrap)} && node --import ./server/node_modules/tsx/dist/loader.mjs ${compiledEntry}`; const command = process.env.PAPERCLIP_INGESTION_SERVER_COMMAND ?? compiledCommand; const autoStartingOnboard = /\bonboard\b[^;&]*(?:--yes|-y)\b/.test(command); if (command.includes("server/dist/index.js") && autoStartingOnboard) { @@ -26,7 +34,7 @@ const providerHome = fs.mkdtempSync(path.join(os.tmpdir(), "ingestion-provider-" export default defineConfig({ ...base, testMatch: "native-event-ingestion.journey.ts", - timeout: 600_000, + timeout: transport === "sandbox-remote" ? 900_000 : 600_000, ...(evidence ? { outputDir: path.join(evidence, "test-results"), reporter: [["list"], ["json", { outputFile: path.join(evidence, "playwright.json") }]] as ReporterDescription[] } : {}), use: { ...base.use, trace: browserRecording ? "on" : "off", video: "off", actionTimeout: 20_000 }, @@ -37,10 +45,11 @@ export default defineConfig({ timeout: 240_000, env: { ...server?.env, + PAPERCLIP_INGESTION_TRANSPORT: transport, PAPERCLIP_STOP_CODEX_COMMAND: path.resolve(import.meta.dirname, "fixtures/native-ingestion-bin/codex"), CODEX_HOME: providerHome, OPENAI_API_KEY: "ingestion-fixture-key", - NODE_OPTIONS: `${server?.env?.NODE_OPTIONS ?? ""} --import=${path.resolve(import.meta.dirname, "fixtures/composer-stop-codex-loader.mjs")}`, + NODE_OPTIONS: `${server?.env?.NODE_OPTIONS ?? ""}${transport === "local" ? ` --import=${path.resolve(import.meta.dirname, "fixtures/composer-stop-codex-loader.mjs")}` : ""}`, }, }, });