fix(runner): address phase 2 tutorial QA

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-07 22:13:41 +00:00
1 parent bfef34393c
commit a9e6db06e0
12 files changed
+215 -48

No files matched your search

+10 -1
View File
@@ -23,6 +23,14 @@ pnpm --filter @paperclipai/paperclip-runner verify
The verification command requires a stable Rust toolchain with `cargo` on
`PATH`, in addition to Node.js 20+ and pnpm 9+.
Minimal Debian/Ubuntu hosts without root access can extract the required
Playwright browser libraries into a user-owned cache and run the same acceptance
sequence with:
```sh
pnpm --filter @paperclipai/paperclip-runner verify:rootless
```
The tracer's final line is stable:
```json
@@ -64,8 +72,9 @@ pnpm --filter @paperclipai/paperclip-runner browser:dev --host 127.0.0.1 --port
| `trace:phase2` | Run one native local session through the Rust runner and fake harness. |
| `record:phase2` | Capture a validated happy-path live trace as a replay fixture. |
| `browser:dev` | Start the standalone live/replay browser devtool. |
| `test:browser` | Exercise static replay and live scenarios, then capture screenshots. |
| `test:browser` | Exercise static replay and live scenarios, then capture temporary screenshots under ignored test output. |
| `verify` | Run the complete Phase 0, Phase 1, and Phase 2 acceptance sequence. |
| `verify:rootless` | Extract Debian/Ubuntu browser libraries without root, then run `verify`. |
## Navigate
@@ -1,10 +1,6 @@
import { expect, test } from "@playwright/test";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
test("validates and renders the shared Phase 1 fixture", async ({ page }) => {
test("validates and renders the shared Phase 1 fixture", async ({ page }, testInfo) => {
await page.goto("/");
await page.getByRole("button", { name: "Static replay" }).click();
await expect(page.getByRole("heading", { name: "Live local runner" })).toBeVisible();
@@ -15,7 +11,7 @@ test("validates and renders the shared Phase 1 fixture", async ({ page }) => {
"The scripted run completed successfully.",
);
await page.screenshot({
path: resolve(packageRoot, "knowledge/evidence/phase-01-static-replay.png"),
path: testInfo.outputPath("phase-01-static-replay.png"),
fullPage: true,
});
});
@@ -32,7 +28,7 @@ test("shows duplicate and unsupported-version replay states", async ({ page }) =
await expect(page.getByText(/protocolVersion 2 is unsupported/)).toBeVisible();
});
test("streams a live run and proves replay parity", async ({ page }) => {
test("streams a live run and proves replay parity", async ({ page }, testInfo) => {
await page.goto("/");
await page.getByRole("button", { name: "Start local run" }).click();
await expect(page.getByTestId("live-status")).toHaveText("Terminal");
@@ -41,12 +37,12 @@ test("streams a live run and proves replay parity", async ({ page }) => {
await expect(page.getByTestId("process-facts")).toContainText("done");
await expect(page.getByTestId("parity-result")).toContainText("Match");
await page.screenshot({
path: resolve(packageRoot, "knowledge/evidence/phase-02-live-complete.png"),
path: testInfo.outputPath("phase-02-live-complete.png"),
fullPage: true,
});
});
test("resolves live permission and input requests", async ({ page }) => {
test("resolves live permission and input requests", async ({ page }, testInfo) => {
await page.goto("/");
await page.getByLabel("Scenario").selectOption("permission-input");
await page.getByRole("button", { name: "Start local run" }).click();
@@ -58,7 +54,7 @@ test("resolves live permission and input requests", async ({ page }) => {
}).first(),
).toContainText("permission: Allow the fake driver to write its local fixture?");
await page.screenshot({
path: resolve(packageRoot, "knowledge/evidence/phase-02-live-permission.png"),
path: testInfo.outputPath("phase-02-live-permission.png"),
fullPage: true,
});
await page.getByRole("button", { name: "Allow" }).click();
@@ -68,13 +64,14 @@ test("resolves live permission and input requests", async ({ page }) => {
hasText: "runtime_request.resolved",
}).first(),
).toContainText("Resolved permission: Allow the fake driver to write its local fixture?");
await page.waitForTimeout(10_500);
await page.getByLabel("Response").fill("phase-02-browser-trace");
await page.getByRole("button", { name: "Send input" }).click();
await expect(page.getByTestId("live-status")).toHaveText("Terminal");
await expect(page.getByTestId("parity-result")).toContainText("Match");
});
test("interrupts a live turn without duplicating terminal state", async ({ page }) => {
test("interrupts a live turn without duplicating terminal state", async ({ page }, testInfo) => {
await page.goto("/");
await page.getByLabel("Scenario").selectOption("interrupted");
await page.getByRole("button", { name: "Start local run" }).click();
@@ -87,7 +84,7 @@ test("interrupts a live turn without duplicating terminal state", async ({ page
await expect(page.getByTestId("terminal-badge")).toHaveAttribute("data-tone", "neutral");
await expect(page.getByTestId("timeline").getByText("run.terminal")).toHaveCount(1);
await page.screenshot({
path: resolve(packageRoot, "knowledge/evidence/phase-02-live-interrupted.png"),
path: testInfo.outputPath("phase-02-live-interrupted.png"),
fullPage: true,
});
});
@@ -39,6 +39,13 @@ The one-command form after installation is:
pnpm --filter @paperclipai/paperclip-runner verify
```
On a minimal Debian or Ubuntu host without root access, use the rootless browser
dependency path:
```sh
pnpm --filter @paperclipai/paperclip-runner verify:rootless
```
## Cumulative guarantees
- the fixture is validated before any mock-core mutation;
@@ -98,8 +98,9 @@ pnpm --filter @paperclipai/paperclip-runner check:browser-tokens
pnpm --filter @paperclipai/paperclip-runner test:browser
```
Expected: two Playwright tests pass and the happy-path screenshot is written to
`packages/paperclip-runner/knowledge/evidence/phase-01-static-replay.png`.
Expected: the Playwright tests pass and temporary screenshots are written under
the ignored `packages/paperclip-runner/test-results/` directory. The regression
suite does not rewrite committed evidence images.
## What this proves
@@ -34,6 +34,16 @@ On a minimal Linux host, install the browser libraries too:
pnpm --filter @paperclipai/paperclip-runner exec playwright install-deps chromium
```
`playwright install-deps` requires root. On a Debian or Ubuntu host where root
is unavailable, use the package's rootless verification command instead. It
downloads the required browser-library packages into a user-owned cache,
extracts them without installing system packages, and scopes `LD_LIBRARY_PATH`
to the verification process:
```sh
pnpm --filter @paperclipai/paperclip-runner verify:rootless
```
## 1. Run the package verification path
```sh
@@ -52,7 +62,7 @@ pnpm --filter @paperclipai/paperclip-runner trace:phase2 -- --scenario happy-pat
Expected summary facts:
- `eventCount` is `18`;
- `eventCount` is `19`;
- `terminalCount` is `1`;
- `semanticResult` is `done`;
- harness and runner exit codes are `0`.
@@ -100,6 +110,8 @@ Open `http://127.0.0.1:4179`, then follow these steps:
4. Confirm `Live and replay reducer output` says `Match`.
5. Select `Permission and input`, start the run, and choose **Allow**.
6. Enter a trace name and choose **Send input**. Confirm the run completes.
Each interactive request remains open for five minutes. If it expires, the
page reports that the run finished and asks you to start a new run.
7. Select `Interruption`, start the run, and choose **Interrupt turn** when the
button becomes active.
8. Confirm the terminal badge says `cancelled`, the semantic result says
@@ -108,6 +120,10 @@ Open `http://127.0.0.1:4179`, then follow these steps:
Stop Vite with `Ctrl+C`.
The browser regression suite writes its temporary screenshots under the
ignored `packages/paperclip-runner/test-results/` directory. It never rewrites
the committed evidence images linked below.
## 5. Inspect the evidence
- [Complete live run](../../knowledge/evidence/phase-02-live-complete.png)
@@ -4,7 +4,7 @@ title: Phase 2 verification
description: Exact tools, commands, results, and screenshots for the local runner and fake harness.
tags: [native-runner, phase-2, verification, rust, supervisor, fake-driver, browser]
status: stable
generated: { by: openai/gpt-5, at: 2026-08-07T21:10:00Z }
generated: { by: openai/gpt-5, at: 2026-08-07T22:13:00Z }
phase: "2"
---
@@ -16,7 +16,7 @@ phase: "2"
- Vitest: `4.1.10`
- Vite: `6.4.1`
- Playwright: `1.61.1`
- Rust/Cargo: `1.97.1` in the run scratch directory
- Rust/Cargo: `1.97.1` in the user-local toolchain
# Commands and observed results
@@ -27,8 +27,8 @@ cargo fmt --manifest-path packages/paperclip-runner/runner/Cargo.toml --all -- -
cargo test --manifest-path packages/paperclip-runner/runner/Cargo.toml --locked --workspace
```
Result: exit 0. Rust formatting passed. Fifteen library tests and one supervisor
integration test passed. The cleanup test proved that the harness and spawned
Result: exit 0. Rust formatting passed. Eighteen library tests and two supervisor
integration tests passed. The cleanup test proved that the harness and spawned
worker process group both stop.
## 2. TypeScript and live-run conformance
@@ -39,11 +39,11 @@ pnpm --filter @paperclipai/paperclip-runner typecheck:typescript
pnpm --filter @paperclipai/paperclip-runner typecheck:browser
```
Result: exit 0. Seven Vitest files and 38 tests passed. The Phase 2 cases cover
Result: exit 0. Eight Vitest files and 46 tests passed. The Phase 2 cases cover
driver lifecycle, command and file items, bounded logs, duplicate commands,
permission/input, interruption, error exit versus semantic result, duplicate
terminal suppression, controller-disconnect cleanup, credentials, and
live/replay parity.
terminal suppression, controller-disconnect cleanup, credentials, live/replay
parity, and the operator-safe rejection of a command sent after completion.
## 3. CLI scenario matrix
@@ -59,11 +59,11 @@ Observed facts:
| Scenario | Events | Terminal | Semantic result | Harness exit | Runner exit |
|---|---:|---:|---|---:|---:|
| happy path | 18 | 1 | `done` | 0 | 0 |
| permission/input | 19 | 1 | `done` | 0 | 0 |
| interrupted | 15 | 1 | `yielded` | 130 | 0 |
| error | 14 | 1 | `yielded` | 7 | 0 |
| duplicate terminal | 14 | 1 | `done` | 0 | 0 |
| happy path | 19 | 1 | `done` | 0 | 0 |
| permission/input | 20 | 1 | `done` | 0 | 0 |
| interrupted | 16 | 1 | `yielded` | 130 | 0 |
| error | 15 | 1 | `yielded` | 7 | 0 |
| duplicate terminal | 15 | 1 | `done` | 0 | 0 |
The duplicate turn command returned one duplicate receipt and did not create a
second `turn.started` event.
@@ -76,14 +76,30 @@ pnpm --filter @paperclipai/paperclip-runner check:browser-tokens
pnpm --filter @paperclipai/paperclip-runner test:browser
```
Result: exit 0. Vite transformed 219 modules. The token check passed. Five
Playwright tests passed in 3.5 seconds. They cover Phase 1 static replay,
Result: exit 0. Vite transformed 219 modules. The token check passed. Six
Playwright tests passed in 14.3 seconds. They cover Phase 1 static replay,
duplicate/version states, live completion and replay parity, permission/input,
and interruption with one terminal event.
and interruption with one terminal event. The permission/input test deliberately
waited 10.5 seconds before submitting input and still completed with replay
parity.
The minimal host lacked Chromium libraries. The test used Ubuntu packages
extracted under `$PAPERCLIP_RUN_SCRATCH_DIR/playwright-libs` through
`LD_LIBRARY_PATH`; no system library was installed.
The documented development server was also started directly:
```sh
pnpm --filter @paperclipai/paperclip-runner browser:dev --host 127.0.0.1 --port 4179
curl -fsS http://127.0.0.1:4179/
curl -fsS http://127.0.0.1:4179/src/main.tsx
```
Result: both requests returned successfully, the Vite process stayed alive, and
dependency optimization emitted none of the Ajv downlevel-transform errors.
The minimal host lacked Chromium libraries. The rootless helper downloaded
and extracted the matching Ubuntu packages under
`$PAPERCLIP_RUN_SCRATCH_DIR/arm64`, then scoped `LD_LIBRARY_PATH` to
the verification process. No system package was installed and no root access
was used. Playwright screenshots were written under ignored `test-results/`
paths, leaving the committed evidence images unchanged.
## 5. Boundary and knowledge checks
@@ -98,13 +114,13 @@ v0.2 bundle passed validation.
## 6. Complete package acceptance sequence
```sh
pnpm --filter @paperclipai/paperclip-runner verify
pnpm --filter @paperclipai/paperclip-runner verify:rootless
```
Result: exit 0 with the run-scratch Rust and browser-library paths active. The
command built Rust, TypeScript, and browser outputs; ran all typechecks, tests,
goldens, parity, token, boundary, browser, docs, and OKF checks; ran the Phase 0
tracer and Phase 1 replay; and finished with the Phase 2 happy-path summary.
Result: exit 0 with the rootless browser-library path active. The command built
Rust, TypeScript, and browser outputs; ran all typechecks, tests, goldens,
parity, token, boundary, browser, docs, and OKF checks; ran the Phase 0 tracer
and Phase 1 replay; and finished with the Phase 2 happy-path summary.
# Browser artifacts
@@ -112,25 +128,25 @@ tracer and Phase 1 replay; and finished with the Phase 2 happy-path summary.
[Phase 2 completed run screenshot](phase-02-live-complete.png)
- PNG: 1440 × 1786, RGB
- SHA-256: `6c49da63df4de4e2d7e5958bd66cdefa492819474526c345328dcd2b575b887a`
- Shows 18 validated events, process exit 0, semantic result `done`, one terminal
- PNG: 1440 × 1851, RGB
- SHA-256: `01524691337ad58d1b9f4592685a7d3b67d225e1c8aede6afdc576d1a5b5dc15`
- Shows 19 validated events, process exit 0, semantic result `done`, one terminal
event, and live/replay `Match`.
## Permission request
[Phase 2 permission screenshot](phase-02-live-permission.png)
- PNG: 1440 × 1131, RGB
- SHA-256: `192c0b244694d22cfe880e814dade1714ae48ab18a2047aaa2c7714419a0328b`
- PNG: 1440 × 1196, RGB
- SHA-256: `1b33f2345299badef63f8f9ea300aa3d9d46148e5f29aca0b8419c5526f7881d`
- Shows the live pending permission request before resolution.
## Interrupted run
[Phase 2 interrupted run screenshot](phase-02-live-interrupted.png)
- PNG: 1440 × 1591, RGB
- SHA-256: `11003e92e776afe670a4a2c958770b4f844ff373115ff93c3aee1e54238b2bd4`
- PNG: 1440 × 1656, RGB
- SHA-256: `35a6d90e9647bb3ac477c39240652ad9c427c93cec10f2eeada609001d6a45dc`
- Shows exit 130, semantic result `yielded`, cancelled terminal state, one
terminal event, and live/replay `Match`.
+2 -1
View File
@@ -46,7 +46,8 @@
"record:phase2": "pnpm run trace:phase2 -- --quiet --output knowledge/evidence/phase-02-happy-path.json",
"browser:dev": "pnpm run build:typescript && pnpm run build:phase2-binaries && vite --config vite.config.ts",
"browser:preview": "vite preview --config vite.config.ts",
"verify": "pnpm run build && pnpm run typecheck && pnpm run check:phase1-goldens && pnpm run test && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run check:forbidden-imports && pnpm run docs:validate && pnpm run check:phase0-parity && pnpm run check:phase1-parity && pnpm run trace:phase0 && pnpm run replay:phase1 && pnpm run trace:phase2 -- --quiet"
"verify": "pnpm run build && pnpm run typecheck && pnpm run check:phase1-goldens && pnpm run test && pnpm run check:browser-tokens && pnpm run test:browser && pnpm run check:forbidden-imports && pnpm run docs:validate && pnpm run check:phase0-parity && pnpm run check:phase1-parity && pnpm run trace:phase0 && pnpm run replay:phase1 && pnpm run trace:phase2 -- --quiet",
"verify:rootless": "bash scripts/verify-rootless-linux.sh"
},
"devDependencies": {
"@playwright/test": "^1.61.1",
@@ -18,6 +18,8 @@ const RUNNER_STREAM_SCHEMA: &str = "paperclip.runner.stream.v1";
const RUNNER_COMMAND_SCHEMA: &str = "paperclip.prp.command.v1";
const HARNESS_COMMAND_SCHEMA: &str = "paperclip.fake_harness.command.v1";
const HARNESS_MESSAGE_SCHEMA: &str = "paperclip.fake_harness.message.v1";
const HARNESS_COMMAND_TIMEOUT: Duration = Duration::from_secs(10);
const INTERACTIVE_REQUEST_TIMEOUT: Duration = Duration::from_secs(5 * 60);
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Phase2Error(String);
@@ -1327,9 +1329,14 @@ fn receive_harness_command(
command_type: &str,
request_id: Option<&str>,
) -> Result<HarnessCommand, Phase2Error> {
let timeout = if command_type == "request.resolve" {
INTERACTIVE_REQUEST_TIMEOUT
} else {
HARNESS_COMMAND_TIMEOUT
};
loop {
let command = receiver
.recv_timeout(Duration::from_secs(10))
.recv_timeout(timeout)
.map_err(|_| Phase2Error::invalid(format!("timed out waiting for {command_type}")))?;
if command.schema != HARNESS_COMMAND_SCHEMA {
continue;
@@ -0,0 +1,82 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "$(uname -s)" != "Linux" ]]; then
printf 'verify:rootless supports Debian and Ubuntu Linux hosts only.\n' >&2
exit 1
fi
for required_command in apt-cache apt-get dpkg dpkg-deb; do
if ! command -v "$required_command" >/dev/null 2>&1; then
printf 'verify:rootless requires %s (Debian/Ubuntu package tooling).\n' "$required_command" >&2
exit 1
fi
done
rootless_cache_base="${PAPERCLIP_RUNNER_BROWSER_DEPS_DIR:-${PAPERCLIP_RUN_SCRATCH_DIR:-${XDG_CACHE_HOME:-${HOME:?HOME is not set}/.cache}/paperclip-runner/playwright-libs}}"
debian_architecture="$(dpkg --print-architecture)"
rootless_deps_dir="$rootless_cache_base/$debian_architecture"
rootless_marker="$rootless_deps_dir/.complete-v2"
select_available_package() {
local candidate
for candidate in "$@"; do
if apt-cache show "$candidate" >/dev/null 2>&1; then
printf '%s\n' "$candidate"
return 0
fi
done
printf 'No supported package candidate found for: %s\n' "$*" >&2
return 1
}
if [[ ! -f "$rootless_marker" ]]; then
browser_packages=(
"$(select_available_package libatk1.0-0t64 libatk1.0-0)"
"$(select_available_package libatspi2.0-0t64 libatspi2.0-0)"
"$(select_available_package libxcomposite1)"
"$(select_available_package libxdamage1)"
"$(select_available_package libxfixes3)"
"$(select_available_package libxi6)"
"$(select_available_package libxrandr2)"
"$(select_available_package libxrender1)"
"$(select_available_package libgbm1)"
"$(select_available_package libasound2t64 libasound2)"
)
download_dir="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-runner-browser-deps.XXXXXX")"
trap 'rm -rf "$download_dir"' EXIT
mkdir -p "$rootless_deps_dir"
(
cd "$download_dir"
apt-get download "${browser_packages[@]}"
)
shopt -s nullglob
downloaded_packages=("$download_dir"/*.deb)
if [[ ${#downloaded_packages[@]} -ne ${#browser_packages[@]} ]]; then
printf 'Expected %s browser-library packages, downloaded %s.\n' \
"${#browser_packages[@]}" "${#downloaded_packages[@]}" >&2
exit 1
fi
for package_archive in "${downloaded_packages[@]}"; do
# Host applications can inject unrelated runtime libraries (for example an
# embedded database's liblzma). System package tooling must use system libs.
env -u LD_LIBRARY_PATH dpkg-deb --extract "$package_archive" "$rootless_deps_dir"
done
touch "$rootless_marker"
fi
case "$debian_architecture" in
amd64) library_triplet="x86_64-linux-gnu" ;;
arm64) library_triplet="aarch64-linux-gnu" ;;
*)
printf 'verify:rootless does not yet map Debian architecture %s.\n' "$debian_architecture" >&2
exit 1
;;
esac
rootless_library_path="$rootless_deps_dir/usr/lib/$library_triplet:$rootless_deps_dir/lib/$library_triplet:$rootless_deps_dir/usr/lib"
printf 'Running Phase 2 verification with rootless browser libraries from %s.\n' "$rootless_deps_dir"
LD_LIBRARY_PATH="$rootless_library_path${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" pnpm run verify
@@ -143,6 +143,21 @@ describe.sequential("Phase 2 local runner and fake harness", () => {
.toHaveLength(1);
});
it("rejects commands after completion with an operator-safe restart message", async () => {
const handle = await startPhase2Scenario({
scenario: "happy-path",
delayMs: 1,
});
await handle.completion;
await expect(
handle.resolveRequest("request_after_completion", { text: "too late" }),
).rejects.toThrow(
"This request expired because the local run already finished. Start a new run.",
);
});
it("produces the same snapshot for validated live events and replay", async () => {
const trace = await runPhase2Scenario({ scenario: "happy-path", delayMs: 1 });
let live = createPhase2LiveSnapshot(trace);
@@ -336,6 +336,15 @@ class LocalRunnerController implements Phase2RunHandle {
command: PrpCommand,
recordCommand: boolean,
): Promise<Phase2CommandReceipt> {
if (
this.#settled ||
this.#process.stdin.destroyed ||
!this.#process.stdin.writable
) {
return Promise.reject(
new Error("This request expired because the local run already finished. Start a new run."),
);
}
if (recordCommand) {
this.#commands.push(structuredClone(command));
}
+7
View File
@@ -17,6 +17,13 @@ export default defineConfig({
preview: {
host: "127.0.0.1",
},
// Vite's dev dependency optimizer has its own esbuild target. Keep it in
// sync with the standalone browser build so Ajv is never downlevelled.
optimizeDeps: {
esbuildOptions: {
target: "esnext",
},
},
build: {
outDir: resolve(packageRoot, "dist-browser"),
emptyOutDir: true,