From 03dd6ef933af1fc4510d3807e2f07ea1b80b904d Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 13:51:40 -0500 Subject: [PATCH] fix(e2e): admit streamed Pi write paths before control Calibrate partial native arguments against the retained permission boundary, retaining strict conflicting-path rejection. Extend public npm consumer verification through the CLI Pi setup and credential-free packaged startup. Co-Authored-By: Paperclip --- .../2026-10-02-pi-production-readiness.md | 21 +++++- scripts/pi-public-install-probe.mjs | 70 +++++++++++++++++++ scripts/verify-grok-npm-install.mjs | 12 ++++ tests/runner-e2e/FIXTURES.md | 8 ++- tests/runner-e2e/catalog.ts | 3 +- tests/runner-e2e/pi-controls-evidence.test.ts | 27 +++++++ tests/runner-e2e/pi-controls-evidence.ts | 12 +++- 7 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 scripts/pi-public-install-probe.mjs diff --git a/doc/plans/2026-10-02-pi-production-readiness.md b/doc/plans/2026-10-02-pi-production-readiness.md index bede087182..f7ccaea823 100644 --- a/doc/plans/2026-10-02-pi-production-readiness.md +++ b/doc/plans/2026-10-02-pi-production-readiness.md @@ -22,8 +22,8 @@ qualification. The existing draft stack must be reviewed in dependency order. | Live lifecycle | Pending native question survives controller restart in the original process; one creation and resolution; stale answer after provider death rejected; Stop retires owned processes; three warm turns keep process identity | Installed local restart and three-turn warm continuity pass at `f5f57e380`. Provider-death and Stop remain required. | | Product workflows | All 26 explicit Pi Product E2E cells (13 local, 13 Daytona); screenshots, public state, independent artifacts, terminal and cleanup evidence | Historical partial passes exist. Full final-source campaign remains required. | | Runner protocol | Pi roster passes through the native packaged runner and authenticated mock control plane; lifecycle/denial/control cases remain distinct from Product tests | Historical get-task-context pass. Final-source roster remains required. | -| Installed distribution | Public CLI/server tars on ARM Mac, Intel Mac and Linux; normal Pi setup; exact Linux companion and immutable Daytona image imported without binary override | Fresh ARM public installation passes. Linux source/image build is running. Final-source Intel/Linux/Daytona receipts remain required. | -| Governance and spend | Company isolation, human-only permission, duplicate/stale answers, Stop and budget hard stop; pricing estimates never become claimed bills | Existing focused invariants need final-source verification. The qualification key cap is not proof of Paperclip budget enforcement. | +| Installed distribution | Public CLI/server tars on ARM Mac, Intel Mac and Linux; normal Pi setup; exact Linux companion and immutable Daytona image imported without binary override | Fresh ARM public installation and normal packaged no-key probe pass. The Linux fleet job is queued. Final-source Intel/Linux/Daytona receipts remain required. | +| Governance and spend | Company isolation, human-only permission, duplicate/stale answers, Stop and budget hard stop; pricing estimates never become claimed bills | 58 focused governance/cost/session invariants pass at `f5f57e380`. Live Stop remains required. The qualification key cap is not proof of Paperclip budget enforcement. | | Integration and rollout | Review each prerequisite; final-head typecheck, tests, build and CI pass; no unresolved review; exact artifacts; rollback recorded | Draft #14956 is stacked on #14924. Prerequisite PRs #14921–#14924 remain open. No merge or release. | ## Bounded execution @@ -61,6 +61,23 @@ prices are estimates. Unpriced usage must stay unpriced in the ledger and UI. - Credential-free ARM startup through the installed CLI, server and database passed. Normal Pi setup verifies the profile-13 closure. Full transport and recovery regressions at `780471702` passed 216 tests. +- At `f5f57e380`, the explicit Runner `get-task-context` case passes. The next + case, `context-before-action`, times out after 120 seconds: four context tools + succeed, but the requested progress mutation and terminal do not arrive. + Its canonical timeout grade remains unchanged; no automatic retry occurred. +- The local pending-permission Stop attempt at `f5f57e380` fails. Retained + Product events contain the native write and pending permission. The oracle + incorrectly rejects legitimate earlier null paths while arguments stream, + so it never sends Stop. The correction admits those partial rows only until + the same execution proves the exact path. Missing/conflicting/lost paths and + foreign executions still fail. All 66 control calibrations pass; the failed + paid attempt stays failed and requires a fresh journey after the correction. +- The public-install verifier now packs the public CLI as well as the server, + runs normal explicit Pi setup in its isolated consumer, then proves closed + startup offline. Its standalone ARM probe passes in 7.9 seconds with no + credentials, prompts, binary override or borrowed workspace package. +- Draft #14956 at `2b3d7ff0a` has a fresh Greptile 5/5, the Linux finding is + resolved, and its CI checks pass. Subsequent changes require a fresh review. - Earlier evidence in `doc/architecture/runner-pi-capabilities.md` is historical and must not be counted as qualification of a new source revision. diff --git a/scripts/pi-public-install-probe.mjs b/scripts/pi-public-install-probe.mjs new file mode 100644 index 0000000000..0a55967cd0 --- /dev/null +++ b/scripts/pi-public-install-probe.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env node +// Run in a credential-free public npm consumer after `paperclipai runtime setup pi`. +// Use the server's normal packaged daemon resolver; never submit a prompt. +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readFile, realpath, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const [serverDirectory, ...extra] = process.argv.slice(2); +assert.ok(serverDirectory && extra.length === 0, 'Usage: pi-public-install-probe.mjs INSTALLED_SERVER_ROOT'); +for (const key of Object.keys(process.env)) { + assert.ok(!/(?:API_KEY|TOKEN|SECRET|PASSWORD)$/.test(key), `Unexpected credential variable: ${key}`); +} +assert.equal(process.env.PAPERCLIP_RUNNER_BINARY, undefined); +assert.equal(process.env.NODE_OPTIONS, undefined); +assert.equal(process.env.NODE_PATH, undefined); +const server = await realpath(serverDirectory); +assert.equal(JSON.parse(await readFile(join(server, 'package.json'), 'utf8')).name, '@paperclipai/server'); +const { resolvePaperclipRunnerBinary } = await import(pathToFileURL(join(server, 'dist/services/native-runtime/native-codex-runner.js'))); +const { createCapabilityRunnerdCodexTransport } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/live/runnerd-codex-transport.js'))); +const { QUALIFIED_ACPX_PROFILES } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/drivers/acpx/qualified-profiles.js'))); +assert.equal(QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, 13); +assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest, 'sha256:fe1e6da01b2a9e4c691ca27cf689d2d6de846a93be6b23fc1e103c9addd7b177'); +assert.equal(QUALIFIED_ACPX_PROFILES.pi.qualificationModel, 'openrouter/deepseek/deepseek-v4-flash-0731'); +const daemon = resolvePaperclipRunnerBinary(); +assert.equal(await realpath(daemon), join(server, 'dist/vendor/paperclip-runner/bin/paperclip-runnerd')); +const root = await mkdtemp(join(tmpdir(), 'pi-public-install-probe-')); +const evidence = []; +const started = performance.now(); +const watchdog = setTimeout(() => { console.error('Pi public-install probe exceeded its cleanup deadline'); process.exit(1); }, 85_000); +let bundle; +let failure; +let settledMs; +try { + const workspace = join(root, 'workspace'); await mkdir(workspace); + bundle = createCapabilityRunnerdCodexTransport({ + provider: 'acpx', acpxAgent: 'pi', piThinkingLevel: 'low', acpxPermissionMode: 'deny-all', + runnerBinary: daemon, stateDirectory: join(root, 'state'), + environment: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' }, + onEvidence: value => evidence.push(value), + }); + try { + await bundle.transport.request('initialize', { clientInfo: { name: 'pi-public-install-verification', version: '1' } }); + await bundle.transport.request('thread/start', { + cwd: workspace, model: 'openrouter/deepseek/deepseek-v4-flash-0731', + baseInstructions: 'Credential-free admission probe. No prompt is submitted.', + permissions: 'paperclip-runner-workspace-read-only', dynamicTools: [], + }); + } catch (error) { failure = String(error); } + settledMs = Math.round(performance.now() - started); + await bundle.transport.close(); + assert.match(failure ?? '', /session\.open failed/); + assert.match(failure, /retryable=false, classification=session_ensure_failed/); + assert.doesNotMatch(failure, /timed out/); + assert.ok(settledMs < 60_000, `Pi admission took ${settledMs} ms`); + assert.ok(evidence.some(item => item.runnerExited === true && item.runnerExitCode === 0)); + for (const item of evidence) assert.deepEqual(item.childEnvironmentKeys, ['LANG', 'PATH']); + const state = JSON.parse(await readFile(join(root, 'state/runner/acpx-provider-state.json'), 'utf8')); + assert.equal(state.descriptor.agent, 'pi'); + assert.equal(state.descriptor.agentRuntimeVersion, '1.0.0'); + assert.equal(state.descriptor.piThinkingLevel, 'low'); + assert.equal(state.activeTurnId, null); + assert.equal(state.identity, null); + assert.equal(state.providerExitUnconfirmed, false); + console.log(JSON.stringify({ schema: 'paperclip.pi.public-npm-install.v1', target: `${process.platform}-${process.arch}`, normalPackagedDaemon: true, exactPiProfile: 13, runtime: '1.0.0', credentials: 'none', promptCalls: 0, settledMs, cleanRunnerExit: true })); +} finally { + try { await bundle?.transport.close(); } + finally { clearTimeout(watchdog); await rm(root, { recursive: true, force: true }); } +} diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjs index ee1788f9df..01d0d41647 100644 --- a/scripts/verify-grok-npm-install.mjs +++ b/scripts/verify-grok-npm-install.mjs @@ -30,6 +30,7 @@ try { } } visit('@paperclipai/server'); + visit('paperclipai'); // Match release.sh's unified versioning in temporary staging directories. // Source manifests remain untouched, including independently versioned SDKs. run(process.execPath, [join(repo, 'scripts/build-standalone-public-packages.mjs')], repo); @@ -115,6 +116,17 @@ try { run(process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), prerequisite]); isolated(['node', '/packages/probe.mjs', 'present'], { prerequisite }); console.log(JSON.stringify({ schema: 'paperclip.grok.public-npm-install.v1', sourceRevision, releaseVersion, lifecycleScriptsEnabled: true, lifecycleSentinelVerified: true, lifecycleNetwork: 'none', consumerImage: GROK_PUBLIC_INSTALL_IMAGE, consumerUid, consumerLockPreserved: true, cleanNpmInstall: true, packageCount: needed.size, builtinLauncherPresent: true, separateGrokPackage: false, npmProvisionedBinary: false, missingPrerequisiteRejected: true, provisionedBinaryVerified: true, commandLeaseVerified: true, providerCalls: 0 })); + // Exercise Pi's public CLI and installed server, never a private workspace + // package or binary override. Public dependency downloads are explicit and + // isolated; the actual admission probe runs without a network or credentials. + const piProbe = join(assets, 'pi-public-install-probe.mjs'); + cpSync(join(repo, 'scripts/pi-public-install-probe.mjs'), piProbe); chmodSync(piProbe, 0o644); + const setup = isolated(['node', '/consumer/node_modules/paperclipai/dist/index.js', 'runtime', 'setup', 'pi'], { download: true }).toString(); + const receipt = JSON.parse(setup.trim()); + assert.equal(receipt.status, 'installed_verified'); + assert.equal(receipt.target, 'linux-x64'); + assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), consumerLock, 'Pi setup must preserve the consumer dependency graph'); + console.log(isolated(['node', '/packages/pi-public-install-probe.mjs', '/consumer/node_modules/@paperclipai/server']).toString().trim()); } finally { rmSync(root, { recursive: true, force: true }); } diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 4dd55988a0..f19524206c 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -408,7 +408,7 @@ The explicit-only `pi-controls` suite adds `pending-permission-stop` and `same-turn-steering` on local and Daytona, each with one provider run, a 120-second active-turn timeout and a 300-second attempt budget. These four control cases retain their behavior; the current matrix totals 26 Pi cells. -Pi 1/profile 12 and coverage revisions intentionally change the affected suite +Pi 1/profile 13 and coverage revisions intentionally change the affected suite fingerprints, so older qualification receipts cannot be reused. Catalog presence and deterministic calibration do not constitute paid qualification. @@ -421,6 +421,12 @@ identity mapping. Pi does not emit Cursor/Copilot diagnostic notices; those notices are never synthesized. Earlier native reads can provide orientation; other native operations cannot substitute for the observed write. +Native tool arguments can arrive after the start event. An earlier null target +is allowed only until the same execution first supplies the exact expected +path. Missing targets, conflicting paths, another execution's path, or a later +loss of the proven path fail. The original start and permission rows remain +bound by retained hashes through control dispatch and settlement. + Stop awaits the pending evidence write and rereads that boundary before sending one caller UUID to the public cancel API. It requires the original request's normalized cancellation closure, a cancelled terminal, and the same-scope diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index ebeb5ac105..33fad66c47 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1090,7 +1090,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Pending native-write Stop and browser-originated same-turn steering, with exact control receipts and independent retirement/no-effect evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"), environments: runnerEnvironments, tasks: piControlTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, + definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, + nativeArguments: "streamed-until-exact-target", pending: "paperclip.e2e.pi-control-pending.v1", stop: "paperclip.e2e.pi-stop-settlement.v1", steering: "paperclip.e2e.pi-steering-settlement.v1", permissionPolicy: "approve-reads", lifecycle: "per_turn", normalCompletionProvesStop: false, nativeFollowUp: "not-covered", providerDeath: "not-covered", remoteObservationCoverage: "continuous-through-owned-process-retirement", filesystemAfterRemoteRetirementObserved: false }, diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 8daafed2a4..1cd582f9d1 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -16,8 +16,35 @@ function cancelled() { const binding = { pending, caller: piControlCaller, cancellationRequestId: piCancellationId, dispatchMonotonicNs: String(BigInt(pending.observedMonotonicNs) + 1n) }; return { f, binding, read: () => readPiStopSettlement({ ...f.state(), ...binding }) }; } +function streamedWrite() { + const f = piControlFixture(); + f.events.pop(); + event(f.events[0]!).payload = { ...f.tool, target: null, inputUpdated: false }; + f.append("tool.execution.progressed", { ...f.tool, target: null, inputUpdated: true }); + f.append("tool.execution.progressed", { ...f.tool, inputUpdated: true }); + f.append("runtime_request.created", { request: f.request }); + return f; +} describe("Pi pending control identity", () => { it("accepts Pi native permission/start without invented provider notices", () => expect(piControlFixture().pending()).toMatchObject({ toolCallId: "pi-tool", executionId: "pi-tool", turnId: "turn" })); + it("accepts streamed arguments only after the same execution proves its exact path", () => { + const f = streamedWrite(); + expect(f.pending()).toMatchObject({ startedSourceSeq: 1, requestSourceSeq: 4 }); + }); + it.each([ + ["no complete target", (f: ReturnType) => { event(f.events[2]!).payload.target = null; }], + ["conflicting partial target", f => { event(f.events[1]!).payload.target = "other.txt"; }], + ["target lost after admission", f => { f.append("tool.execution.progressed", { ...f.tool, target: null }); }], + ["different execution supplied target", f => { event(f.events[2]!).payload.executionId = "other"; }], + ["terminal before target", f => { f.events[1]!.eventType = event(f.events[1]!).eventType = "tool.execution.completed"; event(f.events[1]!).payload.status = "failed"; }], + ] satisfies Array<[string, (f: ReturnType) => void]>)("rejects streamed write with %s", (_name, mutate) => { + const f = streamedWrite(); mutate(f); expect(() => f.pending()).toThrow(); + }); + it("retains the partial start hash through actual callback cancellation", () => { + const f = streamedWrite(), pending = f.pending(); f.cancel(); + expect(readPiStopSettlement({ ...f.state(), pending, caller: piControlCaller, cancellationRequestId: piCancellationId, + dispatchMonotonicNs: String(BigInt(pending.observedMonotonicNs) + 1n) })).toMatchObject({ normalCompletionAccepted: false }); + }); it("admits permission-first ACP only after both canonical boundaries exist", () => { const f = piControlFixture(); f.events.reverse(); f.events.forEach((r, i) => { r.seq = event(r).sourceSeq = i + 1; event(r).sourceEventId = `source:run:${i + 1}`; }); diff --git a/tests/runner-e2e/pi-controls-evidence.ts b/tests/runner-e2e/pi-controls-evidence.ts index 6b51e604a0..819e789e78 100644 --- a/tests/runner-e2e/pi-controls-evidence.ts +++ b/tests/runner-e2e/pi-controls-evidence.ts @@ -46,11 +46,17 @@ function origin(events: readonly unknown[], scope: PiControlScope) { const native = rows.filter(x => x.event.eventType.startsWith("tool.execution.") && rec(x.event.payload).transport === "builtin"); const write = native.filter(x => rec(x.event.payload).executionId === executionId); const started = write.filter(x => x.event.eventType === "tool.execution.started"); + // Pi streams native tool arguments. The start and early progress rows can + // have no path yet; the pending permission is admissible only after this + // same execution provides the exact target. Conflicting paths and loss of a + // previously known path still fail, including after denial or cancellation. + const targetIndex = write.findIndex(x => x.event.payload.target === scope.target); requireProof(started.length === 1 && write.filter(x => x.event.eventType === "tool.execution.completed").length <= 1 - && write[0] === started[0] - && write.every(x => ["tool.execution.started", "tool.execution.progressed", "tool.execution.completed"].includes(x.event.eventType) + && write[0] === started[0] && targetIndex >= 0 + && write.every((x, index) => ["tool.execution.started", "tool.execution.progressed", "tool.execution.completed"].includes(x.event.eventType) && x.event.payload.schema === "paperclip.tool.execution.v1" && x.event.payload.name === "write" && x.event.payload.operation === "edit" - && x.event.payload.target === scope.target && x.event.payload.status === (x.event.eventType === "tool.execution.completed" ? "failed" : "running")), "exact native write lifecycle missing"); + && (x.event.payload.target === scope.target || (index < targetIndex && x.event.payload.target === null && x.event.eventType !== "tool.execution.completed")) + && x.event.payload.status === (x.event.eventType === "tool.execution.completed" ? "failed" : "running")), "exact native write lifecycle missing"); requireProof(!write.some((x, index) => x.event.eventType === "tool.execution.completed" && index !== write.length - 1), "write activity after terminal"); // Read-only orientation/bootstrap can precede the write. Never exempt another // edit, shell execution, or unknown native operation as an alleged bootstrap.