From 68b2424a991349ff31fa712c6a384c23c7b50295 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 12:50:51 -0600 Subject: [PATCH] test(perf): freeze qualified compiled ingestion protocol Co-Authored-By: Paperclip --- .../2026-11-03-native-event-ingestion.md | 173 +++++++++++++++++- tests/e2e/fixtures/in-feed-codex.mjs | 30 ++- .../fixtures/native-ingestion-bootstrap.mjs | 10 + tests/e2e/native-event-ingestion.journey.ts | 11 +- .../e2e/playwright-native-ingestion.config.ts | 11 +- 5 files changed, 221 insertions(+), 14 deletions(-) create mode 100644 tests/e2e/fixtures/native-ingestion-bootstrap.mjs diff --git a/doc/plans/2026-11-03-native-event-ingestion.md b/doc/plans/2026-11-03-native-event-ingestion.md index 6697637f09a..9b73ec22a0c 100644 --- a/doc/plans/2026-11-03-native-event-ingestion.md +++ b/doc/plans/2026-11-03-native-event-ingestion.md @@ -58,8 +58,10 @@ Baseline SQL-only investigation: PostgreSQL 18.1, Node 26.4.0, five warm samples per size. The current count/max query examined 1k / 10k / 100k index entries, with median execution times 0.063 / 0.504 / 5.243 ms. This is mechanism evidence only. -Current state: implementation in progress. No candidate speedup or browser result -has been established. Dependencies are installed and the additive migration was +Current state: implementation in progress. No candidate speedup has been +established. The compiled B0 browser fixture passed functional qualification; +paired performance and live-provider acceptance remain outstanding. Dependencies +are installed and the additive migration was generated with the repository command. The lead owns integration and measurement windows; workers own cursor services, benchmark harness and isolated runtime work. @@ -69,3 +71,170 @@ and Stop journeys. Record target, exact revision, provider versus fixture, run IDs, persisted outcomes and accessible evidence. Each PR uses the repository template and includes its own measurements and remaining limitations. Deployed claims require the exact serving revision and a separate matched verification. + +## Reproduction and evidence protocol + +The owner-path harness is `scripts/benchmark-native-event-ingestion.ts`. It loads +the requested baseline owner and changed supporting services from Git, rather +than borrowing the candidate's append implementation. A baseline that already +has cursor projections is prepared with its own pinned rebuild helper. Both +optimized arms must have current projections before steady-state timing. + +```sh +EVENT_BASELINE_REF=2e37c73c05fe84aead1883b72fdf6b1653b58c2a +node --import ./server/node_modules/tsx/dist/loader.mjs scripts/benchmark-native-event-ingestion.ts --baseline-ref="$EVENT_BASELINE_REF" --phase=aa --output=/private/tmp/native-event-aa.json +node --import ./server/node_modules/tsx/dist/loader.mjs scripts/benchmark-native-event-ingestion.ts --baseline-ref="$EVENT_BASELINE_REF" --phase=compare --noise-report=/private/tmp/native-event-aa.json --output=/private/tmp/native-event-cursor.json --require-acceptance +``` + +For B1→B2, run the same A/A command with B1 as `--baseline-ref`, then use +`--phase=batch` and that B1 noise report from the B2 checkout. The default full +matrix uses five alternating pairs, 16 consecutive groups of 32 events per +sample, and ordered/replay/late-gap/warm-source-suffix cases at all three history +sizes. Warm suffixes start above source sequence 1 and retain a contiguous cursor +of zero. Report the actual history before and after each sample; the size label +is the seeded history, not an assertion that appended rows do not accumulate. + +Setup, projection rebuild, warmup, and independent verification stay outside the +timed append interval. Post-sample reads verify durable count, global sequence, +source identity, canonical hash, schema version and exact payload. SQL diagnostics +run separately: actual scan work includes rows removed by filters and index +rechecks. Lock-acquisition query duration includes query/network work; it is not +reported as pure lock wait. Database CPU samples use the backend process with +10 ms CPU-clock resolution. + +`--quick` only qualifies the harness. Acceptance rejects quick runs, A/A runs, +fewer than five pairs, incomplete matrices and unfinished resource cleanup. +Noise reports must match the exact harness hash and workload fingerprint. +`--validate-report= --require-acceptance` additionally checks current +supporting source hashes before reusing successful owner-path evidence. + +### Retained failed calibration + +The first full A/A completed successfully as an experiment but failed the +repeatability requirement. These columns compare the **same baseline with +itself**; they do not demonstrate an optimization: + +| Seeded history / case | Arm A median ms | Arm B median ms | Largest paired relative difference | +| --- | ---: | ---: | ---: | +| 1k / ordered | 233.862 | 287.416 | 88.2% | +| 10k / ordered | 324.838 | 277.984 | 82.5% | +| 100k / ordered | 461.712 | 468.998 | 20.2% | +| 100k / warm source suffix | 640.278 | 770.366 | 125.4% | + +That protocol measured only one 32-event group per sample. The lead approved +16 fixed groups per sample before measuring any candidate, to average short +durable-commit/storage variation. The noise criterion was not relaxed. Late-gap +closure remains a separately seeded structural diagnostic, not a one-operation +latency acceptance gate. A fresh A/A must qualify the revised protocol. + +Original raw evidence is retained at +`/private/tmp/native-event-ingestion-aa-frozen.json` and its `.log`; initial +qualification files are `native-event-ingestion-aa-smoke.json` and +`native-event-ingestion-cursor-smoke.json` in the same directory. These local +paths supplement the inspectable table above and are not final candidate proof. +The original process exited 0 and its owned cluster and temporary source +snapshot were removed. Slow per-company cleanup was diagnosed and removed from +future owned-cluster runs; supplied external test databases still receive scoped +cleanup in dependency order. + +### Compiled application and browser path + +`tests/e2e/playwright-native-ingestion.config.ts` starts an isolated instance, +the compiled server/vendored runner and built static UI. TSX resolves workspace +DB/shared source exports only. This is a local build dimension, not an npm or +hosted deployment qualification. The test records the compiled build stamp, +ingestion/runtime file hashes and a shared fixture/observer protocol hash. + +```sh +PAPERCLIP_E2E_PORT=32762 PAPERCLIP_INGESTION_SMOKE=1 PAPERCLIP_INGESTION_VARIANT=B0-smoke PAPERCLIP_INGESTION_APP_ROOT=/private/tmp/paperclip-native-event-baseline PAPERCLIP_INGESTION_EVIDENCE_DIR=/private/tmp/native-ingestion-browser-smoke node node_modules/@playwright/test/cli.js test --config tests/e2e/playwright-native-ingestion.config.ts +``` + +Omit `PAPERCLIP_INGESTION_SMOKE` for the full matrix. Choose an owned free port +and evidence directory for every execution. The default command configures the +instance through the existing onboarding boundary with `invokedByRun: true`, +then explicitly starts `server/dist/index.js`. Plain `onboard --yes` starts the +source server automatically and cannot serve as preparation for a later compiled +command. The first qualification caught this, was stopped, and its log remains +at `/private/tmp/native-ingestion-browser-b0-smoke-20261103.log`; no application +or performance success was claimed for it. + +The next qualification reached the compiled/static-UI server but timed out at +the 20-second agent-creation request before any measured task began. Its log and +trace remain under `/private/tmp/native-ingestion-browser-b0-smoke2-20261103`. +The fixture now creates ordinary engineer agents, avoiding unrelated CEO default +skill provisioning, with a 120-second setup-only POST deadline. This changes no +streaming or performance acceptance threshold. The cause of the original setup +delay remains unproven until independently isolated; a successful retry alone +does not establish that cause. Owned HTTP and PostgreSQL listeners from that +attempt were confirmed closed. + +The engineer retry passed agent creation and exercised the native runtime, then +exposed a fixture parser defect: the automatically shortened task title contained +a truncated gate URL while the full Markdown description escaped `_` and `=`. +The provider selected the title and rejected that incomplete URL. The saved +failed run reached durable settlement and recorded `native.local_process_stopped`. +Evidence remains under `/private/tmp/native-ingestion-browser-b0-smoke3-20261103`. +The parser now decodes nested input strings, handles Markdown punctuation and +requires a complete loopback gate plan; the UI prompt also starts with an ordinary +human title. A separate protocol qualification reproduced that exact input shape +and emitted all 16 expected deltas. Application rerun remains distinct from that +provider-only proof. + +The fourth compiled B0 qualification passed on baseline +`2e37c73c05fe84aead1883b72fdf6b1653b58c2a`: all 16 foreground deltas were +visible and independently persisted with unique, ordered source identities. The +Stop run acknowledged canonical native cancellation; all seven visible deltas +were independently persisted. Both transcripts survived reload. The foreground +run was `9bf19cd8-7fa9-4a1e-9736-ec1dcab17a17` and the Stop run was +`81ff804f-79e3-438e-9aa0-ff23a7aa9e66`. Evidence, screenshots and the Playwright +report are retained under +`/private/tmp/native-ingestion-browser-b0-smoke4-20261103`; its extracted +`native-ingestion-evidence.json` records the matching source/build revisions, +compiled hot-path hashes and `qualificationOnly: true`. The test passed in +41.5 seconds after startup; other builds were active, so its timings establish +no performance improvement or baseline noise envelope. Owned HTTP 32764 and +PostgreSQL 54334 listeners were confirmed closed. The full workload matrix, +100-run admission, alternating pairs and live-provider walkthrough remain open. + +The provider is deterministic; runtime, persistence, live WebSocket, tools and +UI are real. A separate provider process keeps original intended emission times +even when stdout backpressures. A common gate releases all admitted runs on the +same schedule. The browser snapshots actual text in one animation frame and +timestamps that snapshot in the next, preserving a dirty rescan for later text. +After timing, the test independently reads individual `item.delta` rows for +every foreground/background run and checks count, order and source identities; +completed-item text cannot hide missing deltas. Reload preserves the transcript +and previously visible stopped output. + +The full browser matrix contains paced 1×128 output, 1×16 / 10×128 / 100×512 +bursts, an existing-history case and operator Stop. Existing-history pressure is +**1,000 actual prelude events in the watched run/source**, committed and visible +before releasing the measured phase; background runs have no prelude. The 100k +history case belongs to the owner-path benchmark. It is not claimed for the +browser fixture. + +The policy permits at most 50 concurrent runs per agent; the 100-run fixture uses +two Codex agents configured through the normal API at 50 each. A common gate +requires all requested providers to become ready before emission. Admission +readiness for 100 is a separate qualification obligation; a smaller admitted +count must not be labelled a 100-run measurement or bypassed through direct +database writes. + +Run five alternating baseline/repeat application pairs for A/A and baseline/ +candidate pairs for comparison. A manifest is a JSON array of entries with +`baselineFile` and `candidateFile`, pointing at the corresponding +`native-ingestion-evidence` JSON attachments. Compare without manual arithmetic: + +```sh +node scripts/compare-native-ingestion-browser.ts --phase=aa --pairs=/private/tmp/browser-aa-pairs.json --output=/private/tmp/browser-aa-noise.json +node scripts/compare-native-ingestion-browser.ts --pairs=/private/tmp/browser-ab-pairs.json --noise-report=/private/tmp/browser-aa-noise.json --output=/private/tmp/browser-comparison.json --require-acceptance +``` + +The preregistered primary loaded case is `burst512`: p95 emission/intended-to- +visible latency and drainage through terminal settlement must improve beyond +A/A noise in at least four of five pairs. Other case metrics are nonregression +controls. Use `--expect=nonregression` for the cursor-only application control; +the combined change still requires the loaded improvement. Comparison rejects +smoke, source-server evidence, mismatched artifacts/protocols, missing background +delta proof and incomplete runs. Live-provider walkthroughs remain a separate +integration gate. diff --git a/tests/e2e/fixtures/in-feed-codex.mjs b/tests/e2e/fixtures/in-feed-codex.mjs index 0f1a44c47c4..ced943c1e35 100644 --- a/tests/e2e/fixtures/in-feed-codex.mjs +++ b/tests/e2e/fixtures/in-feed-codex.mjs @@ -27,6 +27,29 @@ const call = (tool, args) => new Promise((resolve, reject) => { async function sendIngestion(value) { if (!process.stdout.write(`${JSON.stringify(value)}\n`)) await once(process.stdout, 'drain'); } +function ingestionFixturePlan(value) { + const strings = []; + function collect(input, depth = 0) { + if (depth > 6) return; + if (typeof input === 'string') { + strings.push(input); + try { collect(JSON.parse(input), depth + 1); } catch { /* Ordinary prompt text. */ } + } else if (Array.isArray(input)) for (const item of input) collect(item, depth + 1); + else if (input && typeof input === 'object') for (const item of Object.values(input)) collect(item, depth + 1); + } + collect(value); + for (let text of strings) { + // The real Markdown composer escapes punctuation, and the native brief + // also carries a shortened title. Accept a complete fixture plan only. + for (let pass = 0; pass < 4; pass++) text = text.replace(/\\([\\_=:./-])/g, '$1'); + const match = text.match(/INGESTION_PERF nonce=([a-z0-9]+) events=(\d+) intervalMs=(\d+) prelude=(\d+) gate=(http:\/\/(?:127\.0\.0\.1|localhost):\d+\/native-ingestion-gate\/[a-f0-9]{32})/); + if (!match) continue; + const plan = { nonce: match[1], events: Number(match[2]), intervalMs: Number(match[3]), prelude: Number(match[4]), gate: match[5] }; + if (plan.events < 1 || plan.events > 4096 || plan.intervalMs > 1000 || plan.prelude > 10_000) throw new Error('Ingestion fixture exceeds its bounded workload'); + return plan; + } + throw new Error('Missing complete INGESTION_PERF fixture plan'); +} async function streamIngestion() { if (!ingestionPlan) throw new Error('Missing INGESTION_PERF fixture plan'); @@ -220,12 +243,7 @@ createInterface({ input: process.stdin }).on('line', (line) => { helloProbe = (process.argv.includes('--completion-stream-fixture') || ingestionFixture) && JSON.stringify(message.params?.input).includes('Respond only with hello. Do not use tools or inspect files.'); if (ingestionFixture && !helloProbe) { - const match = JSON.stringify(message.params?.input).match(/INGESTION_PERF nonce=([a-z0-9]+) events=(\d+) intervalMs=(\d+) prelude=(\d+) gate=([^\s"\\]+)/); - if (!match) throw new Error('Missing explicit ingestion fixture plan'); - ingestionPlan = { nonce: match[1], events: Number(match[2]), intervalMs: Number(match[3]), prelude: Number(match[4]), gate: match[5] }; - if (ingestionPlan.events < 1 || ingestionPlan.events > 4096 || ingestionPlan.intervalMs > 1000 || ingestionPlan.prelude > 10_000) { - throw new Error('Ingestion fixture exceeds its bounded workload'); - } + ingestionPlan = ingestionFixturePlan(message.params?.input); } emitCeoLineage = JSON.stringify(message.params).includes('CEO descendant fixture'); declined = /connection_intent/.test(JSON.stringify(message.params)) && /rejected/.test(JSON.stringify(message.params)); diff --git a/tests/e2e/fixtures/native-ingestion-bootstrap.mjs b/tests/e2e/fixtures/native-ingestion-bootstrap.mjs new file mode 100644 index 00000000000..550f4ed64c6 --- /dev/null +++ b/tests/e2e/fixtures/native-ingestion-bootstrap.mjs @@ -0,0 +1,10 @@ +// Configure the disposable instance through the same onboarding boundary used +// by `run`, without onboarding's --yes foreground auto-start. The next command +// starts server/dist/index.js explicitly so measured hot paths stay compiled. +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +if (process.env.NODE_ENV !== 'test') throw new Error('Ingestion bootstrap is only for an isolated test instance'); +if (!process.env.PAPERCLIP_HOME || !process.env.PAPERCLIP_CONFIG) throw new Error('Missing isolated instance paths'); +const { onboard } = await import(pathToFileURL(path.resolve('cli/src/commands/onboard.ts')).href); +await onboard({ yes: true, bind: 'loopback', installService: false, invokedByRun: true }); diff --git a/tests/e2e/native-event-ingestion.journey.ts b/tests/e2e/native-event-ingestion.journey.ts index cecd73d2338..e9c35cc3719 100644 --- a/tests/e2e/native-event-ingestion.journey.ts +++ b/tests/e2e/native-event-ingestion.journey.ts @@ -107,7 +107,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy } report.fixtureSha256 = createHash("sha256").update(await readFile(path.resolve(import.meta.dirname, "fixtures/in-feed-codex.mjs"))).digest("hex"); const protocolFiles = ["native-event-ingestion.journey.ts", "playwright-native-ingestion.config.ts", "fixtures/in-feed-codex.mjs", - "fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs"]; + "fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs"]; report.measurementSha256 = createHash("sha256").update(Buffer.concat(await Promise.all(protocolFiles.map(file => readFile(path.resolve(import.meta.dirname, file)))))).digest("hex"); try { await json(await request.patch("/api/instance/settings/experimental", { data: { enableNativeRunner: true, enableWorktreeRunExecution: true } })); @@ -116,8 +116,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const agents = []; const agentCount = Math.ceil(Math.max(...cases.map(item => item.runs)) / 50); for (let index = 0; index < agentCount; index++) { - const agent = await json(await request.post(`/api/companies/${company.id}/agents`, { data: { - name: `Ingestion ${index + 1}`, role: index === 0 ? "ceo" : "engineer", adapterType: "paperclip_runner", + const agent = await json(await request.post(`/api/companies/${company.id}/agents`, { timeout: 120_000, data: { + name: `Ingestion ${index + 1}`, role: "engineer", adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-5.1-codex-mini", env: { OPENAI_API_KEY: { type: "user_secret_ref", key: "OPENAI_API_KEY", version: "latest", required: true }, } }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true, maxConcurrentRuns: 50 } }, @@ -130,7 +130,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy async function perform(workload: Case, stop = false) { const key = randomUUID().replaceAll("-", ""), nonce = randomUUID().replaceAll("-", ""); pending.set(key, new Map()); - const prompt = (token: string, prelude: number) => `INGESTION_PERF nonce=${token} events=${workload.events} intervalMs=${workload.intervalMs} prelude=${prelude} gate=${base}${key}`; + const prompt = (token: string, prelude: number) => `Measure native streaming for ${token.slice(0, 8)}.\n\nINGESTION_PERF nonce=${token} events=${workload.events} intervalMs=${workload.intervalMs} prelude=${prelude} gate=${base}${key}`; const before = new Set((await json(await request.get(`/api/companies/${company.id}/issues`))).map((issue: { id: string }) => issue.id)); await page.goto(`/${company.issuePrefix}/agents/${agents[0].id}/overview`); await page.getByRole("link", { name: "Tasks", exact: true }).click(); @@ -180,6 +180,9 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy return runs.length === issueIds.length && runs.every(run => ["succeeded", "cancelled", "failed"].includes(run.status)); }, { timeout: 180_000 }).toBe(true); const settledAt = Date.now(), visible = await observed(); + const details = []; + for (const run of runs) details.push(await json(await request.get(`/api/heartbeat-runs/${run.id}`))); + runs = details; expect(runs.every(run => run.runtimeMode === "native")).toBe(true); expect(runs.every(run => run.status === (stop ? "cancelled" : "succeeded"))).toBe(true); if (stop) await expect.poll(async () => diff --git a/tests/e2e/playwright-native-ingestion.config.ts b/tests/e2e/playwright-native-ingestion.config.ts index 981d719fca5..c76b3fdcb8d 100644 --- a/tests/e2e/playwright-native-ingestion.config.ts +++ b/tests/e2e/playwright-native-ingestion.config.ts @@ -7,8 +7,15 @@ import base from "./playwright.config"; const server = base.webServer as Exclude; const evidence = process.env.PAPERCLIP_INGESTION_EVIDENCE_DIR; const appRoot = path.resolve(process.env.PAPERCLIP_INGESTION_APP_ROOT ?? path.resolve(import.meta.dirname, "../..")); -const command = process.env.PAPERCLIP_INGESTION_SERVER_COMMAND ?? server?.command; -const compiled = Boolean(command?.includes("server/dist/index.js") && !command.includes("server/src/index") && !/onboard[^;&]*--run/.test(command)); +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; +const bootstrap = path.resolve(import.meta.dirname, "fixtures/native-ingestion-bootstrap.mjs"); +const compiledCommand = `node --import ./server/node_modules/tsx/dist/loader.mjs ${quote(bootstrap)} && node --import ./server/node_modules/tsx/dist/loader.mjs server/dist/index.js`; +const command = process.env.PAPERCLIP_INGESTION_SERVER_COMMAND ?? compiledCommand; +const autoStartingOnboard = /\bonboard\b[^;&]*(?:--yes|-y)\b/.test(command); +if (command.includes("server/dist/index.js") && autoStartingOnboard) { + throw new Error("onboard --yes starts the source server before the compiled command; use native-ingestion-bootstrap.mjs for configuration only"); +} +const compiled = Boolean(command.includes("server/dist/index.js") && !command.includes("server/src/index") && !autoStartingOnboard); if (process.env.PAPERCLIP_INGESTION_SMOKE !== "1" && !compiled) { throw new Error("Full ingestion measurements require a compiled server/dist/index.js command; source CLI is qualification smoke only"); }