From 5cd6d3d5237e3e15394fcc1c7e754c30a63c5169 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 1 Oct 2026 17:36:20 -0500 Subject: [PATCH] test: strengthen Pi editing and pending native recovery qualification --- doc/evals.md | 6 +- tests/runner-e2e/FIXTURES.md | 63 +++++- tests/runner-e2e/README.md | 9 +- tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 10 +- tests/runner-e2e/pi-controls-evidence.test.ts | 18 +- tests/runner-e2e/pi-file-evidence.test.ts | 182 +++++++++++++++++ tests/runner-e2e/pi-file-evidence.ts | 192 ++++++++++++++++++ tests/runner-e2e/pi-native-cases.test.ts | 10 +- tests/runner-e2e/pi-native-cases.ts | 6 + tests/runner-e2e/pi-native-flow.ts | 13 +- .../runner-e2e/pi-native-restart-flow.test.ts | 152 ++++++++++++++ tests/runner-e2e/pi-native-restart-flow.ts | 158 ++++++++++++++ tests/runner-e2e/runner.spec.ts | 16 ++ tests/runner-e2e/server-stop.test.ts | 43 +++- 15 files changed, 852 insertions(+), 32 deletions(-) create mode 100644 tests/runner-e2e/pi-file-evidence.test.ts create mode 100644 tests/runner-e2e/pi-file-evidence.ts create mode 100644 tests/runner-e2e/pi-native-restart-flow.test.ts create mode 100644 tests/runner-e2e/pi-native-restart-flow.ts diff --git a/doc/evals.md b/doc/evals.md index 53dcfd52eb5..87410e8e1f7 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -35,8 +35,10 @@ Pi's explicit-only `pi-controls` Product suite tests Stop while native permissio is unanswered and browser-originated same-turn steering, locally and on Daytona. Its [fixture contract](../tests/runner-e2e/FIXTURES.md#pi-active-controls) separates control acknowledgment, actual message consumption and owned process retirement. -The four cells add coverage without changing the existing 19 Pi cells; they -remain unqualified until measured on the exact candidate runtime. +The current Pi matrix has 25 explicit cells (13 local and 12 Daytona), including +these four controls and pending-native-question controller restart. File editing +also requires native edit/validation events and a registered artifact download. +All remain unqualified until measured on the exact candidate runtime and harness. Use **Runner Evals** for a runner protocol, adapter, transport, native session, tool grant, or one-turn provider qualification question. The workflow checks diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index a77628cf610..f360da075fc 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -286,7 +286,7 @@ is explicit-only. No private control-plane hooks or direct database writes are u ## Pi native boundaries -The explicit-only `pi-native` suite has four local and three Daytona candidate +The explicit-only `pi-native` suite has five local and four Daytona candidate cells, with no automatic retries. The remote suite excludes automatic deny-all because its initial native file read is itself denied. `native-questions` answers the real runner-owned Pi select, confirm, input and editor tool through four durable browser cards, reloading before every answer. @@ -304,12 +304,69 @@ is reconnect evidence only; these cases do not establish provider-death recovery They use public product APIs, real browser answers, and ordinary isolated files; no database writes, private hooks, or fabricated provider results are allowed. +### Pending native controller restart + +`native-pending-controller-restart` restarts the public controller while one Pi +input callback remains unanswered. It requires the same durable interaction, +request, live run, native session, turn and producer before and after restart. +Only then does the browser submit previously undisclosed text. One durable +resolution, one original successful turn and independently read exact workspace +JSON prove delivery. A replacement run, replay, cancellation, expiry, rewritten +request or merely reloaded browser cannot pass. Full states and PRP identities +stay in private snapshots under the existing publication allowlist. + +### Pi file editing and registered artifacts + +Pi's `extended-harnesses/file-edit-validate` seeds exact bytes before startup and +requires one native edit lifecycle followed by a successful native bash execution +with the exact nonce-bound byte-validation command as its projected title and a +validation marker. A marker-only echo cannot pass. Independent final bytes must +match the fixture. The +real `register_deliverable` receipt, attachment metadata, publication activity, +visible task attachment and authenticated public download must all agree on the +file's bytes, hash, company, issue, agent and originating run. A file on disk or a +model completion claim cannot substitute for publication. Daytona additionally +requires the public run's finalized `nativeWorkspaceSync` descriptor, baseline and +final-host hashes, bound workspace and public environment lease to match this +run/company/environment/provider lease and remote root. The checked host bytes +come through production stage-in/copy-back; this is explicitly `product_copyback` +provenance, not a sealed guest observation or an independently recomputed whole +host-workspace snapshot. Downloaded artifact bytes remain independently checked. + +Private `pi-file-seed.json`, `pi-file-observation.json` and `pi-file-evidence.json` +retain the seed, downloaded/workspace bytes, public publication receipts, checked hashes, +correlated tool identities and a before/after diff computed from independently +checked workspace bytes. This is not native diff presentation. The current +common tool projection omits raw arguments and reports typed `exitCode: null`; +the oracle checks the command title and completed lifecycle, without claiming raw +invocation arguments or a typed exit code. The sidecar converts Pi absolute file +locations into bounded workspace-relative display targets; the oracle requires +the exact relative filename, but that display value is not file-access authority. +Restoring raw arguments, typed exit code and native diff presentation remains a +capability follow-up. + +Wrong or missing lifecycle, byte, registration, download or recovery evidence is +a candidate failure; transport/infrastructure failures retain the existing +harness classification. Positive and plausible-wrong/missing-evidence unit +calibrations do not count as paid qualification. All automatic retries stay zero. + +Provider-death coverage remains bounded: the existing patched-ACP package test +`provider exit fails promptly and missing owned extension fails admission` runs +an actual wrapper child against a deterministic fake Pi child. The bridge test +`expires pending requests and discards a late answer exactly once` and driver +abort/event-stream tests separately cover pending cleanup with mocked transport. +They do not establish a single actual native-provider-death to durable browser +request-expiry chain. No local or Daytona paid death case is claimed, and no +arbitrary process-kill API is introduced by these fixtures. + ## Pi active controls The explicit-only `pi-controls` suite adds `pending-permission-stop` and `same-turn-steering` on local and Daytona, each with one provider run, a -120-second active-turn timeout and a 300-second attempt budget. The existing -19 Pi cells and their suite fingerprints are unchanged. Catalog presence and +120-second active-turn timeout and a 300-second attempt budget. These four +control cases retain their behavior; the current matrix totals 25 Pi cells. +Pi 1/profile 11 and coverage revisions intentionally change the affected suite +fingerprints, so older qualification receipts cannot be reused. Catalog presence and deterministic calibration do not constitute paid qualification. Both cases create a task through the browser and select `approve-reads` and diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 42be5397c91..6f41d46ea95 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -39,7 +39,9 @@ native-write evidence, caller-owned control acknowledgments, independent no-effect/retirement proof and the actual production task UI. Daytona also requires the existing immutable image and executable digests. These cells do not establish native follow-up queue persistence or provider-death recovery. -They do not change the existing 19 Pi cells or promote Pi's pending profile. +The current Pi matrix has 25 explicit cells (13 local and 12 Daytona), including +these four controls and the pending-native-question restart cases. Pi's profile +remains pending; catalog presence is not live qualification. ## Provider-free browser bootstrap regression @@ -1435,8 +1437,9 @@ mock control plane. Neither suite substitutes for the other. ### Pi native Product fixtures -The separate `pi-native` suite defines seven explicit Pi cells: native questions, -agent-file persistence and browser permission denial on local and Daytona, plus +The separate `pi-native` suite defines nine explicit Pi cells: native questions, +agent-file persistence, browser permission denial and controller restart with an +unanswered native question on local and Daytona, plus a local-only automatic `restrictive-denial` case. The latter cannot use the remote setup read under `deny-all`, so it is explicitly excluded there. It uses the pinned candidate profile and exact OpenRouter model. It is excluded from `--all`, diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 125179ac420..02d8f0c4b5a 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -136,10 +136,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 4, 8, 7, 4, 4, 6, 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, + 4, 8, 9, 4, 4, 6, 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, ]); - expect(validateRunnerCatalog()).toHaveLength(466); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(466); + expect(validateRunnerCatalog()).toHaveLength(468); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(468); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index e87a83fdb32..0cf9d6ec813 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -3,6 +3,7 @@ import { piControlTasks } from "./pi-controls-cases.js"; import { cursorNativeTasks } from "./cursor-native-cases.js"; import { copilotProtectionTasks } from "./copilot-protection-tasks.js"; import { piNativeTasks } from "./pi-native-cases.js"; +import { piFilePrompt } from "./pi-file-evidence.js"; import { chatConfirmationTasks } from "./chat-cases.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; import { apiResponseReadingTask } from "./api-response-reading.js"; @@ -1102,9 +1103,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ id: "pi-native", label: "Pi native boundaries", manualOnly: true, description: "Pi native forms, registered agent files and human permission denial on local and Daytona execution; automatic deny-all remains local-only.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"), - environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 7, + environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 9, excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial"], - definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer" }, + definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request" }, }, { id: "native-active-stop", label: "Stop an unanswered native permission", manualOnly: true, @@ -1146,7 +1147,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments, tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask], expectedMatrixSize: 30, - definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28" }, + definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", piFileEvidence: "seed-edit-execute-public-download-v1" }, }, { id: "instruction-persistence", label: "Instruction Persistence", @@ -1489,7 +1490,8 @@ export function buildRunnerMatrix( suiteDefinitionHash: suiteDefinitionHash(suite), profile, environment, - task, + task: suite.id === "extended-harnesses" && profile.qualificationCandidate === "pi" && task.id === "file-edit-validate" + ? { ...task, buildPrompt: piFilePrompt } : task, groups: [ ...new Set([ ...suite.groups, diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index d30f5685454..dcf2238ec1c 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -124,18 +124,24 @@ describe("Pi controls catalog admission", () => { expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true); expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false); - expect(validateRunnerCatalog()).toHaveLength(466); + expect(validateRunnerCatalog()).toHaveLength(468); for (const cell of cells) expect(JSON.parse(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "pi", model: cell.profile.model }]); expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); - it("keeps existing Pi-native and active Stop definition identities", () => { + it("pins the Pi 1 profile and versioned coverage while retaining active Stop identity", () => { + // Pi 1/profile 11 changes profile-bearing definitions. Coverage v3/v2 adds + // pending restart and the strict file oracle; no runtime admission is promoted. + const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; + expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 3, profileVersion: 11 }); + expect(runnerSuites.find(s => s.id === "extended-harnesses")!.definitionMetadata).toMatchObject({ version: 2 }); const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "7aaacf83924fdfc7163594e08b9a64ac516f9eb8a0c2c96636b6d2fe4f23776a", + "pi-native": "00d4edfd0d61772977434389cb601bbdc08173db4a33fd1e153d2e18f488f597", "native-active-stop": "99682b2b106d816a011834fae5a944ed7729958893709d5b83a19b6f595e7e4d", - "rich-acp-warm-continuity": "aead43b3f4f8ed285e2d1c30a517a1850633ae42e593f53122bea9e72af92585", - "extended-harnesses": "e75cb2222d2f52ec3a2a425211d195d36b95882de2cb9ed0ad93231b7c42768a", + "rich-acp-warm-continuity": "25e69f031696aeb459e6722a056ae0802379f9137799648fdd1945881f32f41a", + "extended-harnesses": "e50f79cf604bffb2cdee3ad4e8bc1d106045a4f44ef07e877032ff5b8960c8ab", }); - expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi" && c.suite.id !== "pi-controls")).toHaveLength(19); + expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi" && c.suite.id !== "pi-controls")).toHaveLength(21); }); }); diff --git a/tests/runner-e2e/pi-file-evidence.test.ts b/tests/runner-e2e/pi-file-evidence.test.ts new file mode 100644 index 00000000000..03cabdff824 --- /dev/null +++ b/tests/runner-e2e/pi-file-evidence.test.ts @@ -0,0 +1,182 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { collectPiFileEvidence, gradePiCopyback, gradePiFileEvidence, piFileContract, seedPiFile } from "./pi-file-evidence.js"; +import { canonicalProviderEventsFromAcpxRuntimeEvent } from "../../packages/paperclip-runner/src/provider-events.js"; +import { safeAcpxLocations } from "../../packages/paperclip-runner/src/drivers/acpx/safe-locations.js"; +import { classifyFailure } from "./failure-classifier.js"; +import { runnerMatrix } from "./catalog.js"; + +function fixture() { + const c = piFileContract("fixture"), attachmentId = "12345678-1234-1234-1234-123456789abc"; + const tool = (seq: number, executionId: string, name: string, operation: string, completed: boolean, target: string | null, output = "") => { + // Pi emits absolute edit locations and uses the bash command as title. + // Exercise sidecar path normalization followed by the common projection; + // direct raw ACP projection would incorrectly lose the workspace target. + const canonical = canonicalProviderEventsFromAcpxRuntimeEvent({ type: "tool_call", toolCallId: executionId, + tag: completed ? "tool_call_update" : "tool_call", title: name, kind: operation, + status: completed ? "completed" : "pending", locations: safeAcpxLocations(target ? [{ path: `/workspace/${target}` }] : [], "/workspace", operation, name), rawOutput: output }, executionId, "turn")[0]!; + return { companyId: "company", runId: "run", seq, protocolSchemaVersion: 1, eventType: canonical.eventType, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "runner", + sourceSeq: seq, sourceEventId: `runner:run:${seq}`, runId: "run", turnId: "turn", normalizedSessionId: "session", ...canonical } } }; + }; + return { + environment: "local" as const, environmentId: "environment", nonce: "fixture", companyId: "company", issueId: "issue", agentId: "agent", attachmentId, + seed: { schema: "paperclip.e2e.pi-file-seed.v1", filename: c.filename, + beforeSha256: createHash("sha256").update("ready-fixture\n").digest("hex"), byteSize: Buffer.byteLength(c.before) }, + workspaceBytes: Buffer.from(c.after), downloadedBytes: Buffer.from(c.after), + run: { id: "run", companyId: "company", agentId: "agent", runtimeMode: "native", nativeSessionId: "session", runnerInstanceId: "runner", status: "succeeded", resultJson: { semanticToolReceipts: { + publish: { operationId: "register_deliverable", input: { filename: c.filename, contentRef: c.filename, contentType: "text/plain", byteSize: c.byteSize, sha256: c.sha256 }, + result: { disposition: "applied", commandId: `deliverable-prepared:${attachmentId}`, entityRefs: [attachmentId, "product", "comment"] } }, + } } }, + events: [tool(1, "edit", "edit", "edit", false, c.filename), tool(2, "edit", "edit", "edit", true, c.filename), + tool(3, "validate", c.validationCommand, "execute", false, null), tool(4, "validate", c.validationCommand, "execute", true, null, c.validationMarker)], + attachments: [{ id: attachmentId, companyId: "company", issueId: "issue", originatingRunId: "run", createdByAgentId: "agent", + originalFilename: c.filename, contentType: "text/plain", byteSize: c.byteSize, sha256: c.sha256 }], + activity: [{ action: "issue.attachment_added", companyId: "company", runId: "run", actorId: "agent", entityId: "issue", + details: { attachmentId, source: "paperclip_runner_protocol" } }], + }; +} +describe("Pi edit, validation and public artifact oracle", () => { + it("accepts actual correlated lifecycles, independent bytes and a run-bound registered download", () => { + const result = gradePiFileEvidence(fixture()); + expect(result.passed).toBe(true); + expect(result.diff.text).toContain("-ready-fixture\n+verified-fixture\n"); + expect(result.limits.join(" ")).toContain("not attested"); + expect(result.verification.providerExecutionStatus).toBe("completed"); + expect(result.verification.commandTitle).toBe(piFileContract("fixture").validationCommand); + expect(result.verification.nativeFileAttribution).toBe("workspace_relative_display_target"); + }); + const mutations: Array<[string, (f: ReturnType) => void]> = [ + ["absent seed", f => { f.seed = {} as any; }], + ["wrong final bytes", f => { f.workspaceBytes = Buffer.from("wrong\n"); }], + ["final-only write", f => { f.events[0]!.payload.prpEvent.payload.name = "write"; f.events[1]!.payload.prpEvent.payload.name = "write"; }], + ["missing edit", f => { f.events = f.events.slice(2); }], + ["missing validation", f => { f.events = f.events.slice(0, 2); }], + ["echo-only validation", f => { f.events[2]!.payload.prpEvent.payload.name = `echo ${piFileContract("fixture").validationMarker}`; f.events[3]!.payload.prpEvent.payload.name = f.events[2]!.payload.prpEvent.payload.name; }], + ["failed validation", f => { f.events[3]!.payload.prpEvent.payload.status = "failed"; }], + ["unfinished validation", f => { f.events.pop(); }], + ["unrelated validation turn", f => { f.events[2]!.payload.prpEvent.turnId = "other"; f.events[3]!.payload.prpEvent.turnId = "other"; }], + ["foreign native session", f => { f.run.nativeSessionId = "other"; }], + ["foreign native producer", f => { f.run.runnerInstanceId = "other"; }], + ["foreign event run", f => { f.events[1]!.payload.prpEvent.runId = "other"; }], + ["duplicate event", f => { f.events.push(structuredClone(f.events[3]!)); }], + ["missing tool start", f => { f.events.splice(2, 1); }], + ["truncated validation output", f => { f.events[3]!.payload.prpEvent.payload.outputTruncated = true; }], + ["missing native target", f => { f.events[0]!.payload.prpEvent.payload.target = null; f.events[1]!.payload.prpEvent.payload.target = null; }], + ["wrong target", f => { f.events[1]!.payload.prpEvent.payload.target = "other.txt"; }], + ["missing registration", f => { f.run.resultJson.semanticToolReceipts = {} as any; }], + ["rejected registration", f => { f.run.resultJson.semanticToolReceipts.publish.result.disposition = "denied"; }], + ["wrong registered hash", f => { f.run.resultJson.semanticToolReceipts.publish.input.sha256 = "wrong"; }], + ["wrong registered entity", f => { f.run.resultJson.semanticToolReceipts.publish.result.entityRefs[0] = "other"; }], + ["missing attachment", f => { f.attachments = []; }], + ["foreign attachment company", f => { f.attachments[0]!.companyId = "other"; }], + ["stale attachment run", f => { f.attachments[0]!.originatingRunId = "older"; }], + ["wrong download bytes", f => { f.downloadedBytes = Buffer.from("wrong\n"); }], + ["missing publication activity", f => { f.activity = []; }], + ["foreign publication run", f => { f.activity[0]!.runId = "other"; }], + ]; + it.each(mutations)("rejects %s even when the model claims completion", (_name, change) => { + const f = fixture(); change(f); expect(() => gradePiFileEvidence(f)).toThrow("Pi file evidence:"); + try { gradePiFileEvidence(f); } catch (error) { expect(classifyFailure(error)).toBe("candidate_failure"); } + }); + it("keeps real nonce commands within the Rust 240-character name contract", () => { + const c = piFileContract("0123456789ab-1"); + expect([...c.validationCommand].length).toBeLessThanOrEqual(240); + expect(safeAcpxLocations([{ path: `/workspace/${c.filename}` }], "/workspace", "edit", "edit")[0]).toMatchObject({ + path: c.filename, pathBoundary: "paperclip.workspace_relative_display.v2", + }); + expect(() => piFileContract("a".repeat(100))).toThrow("native Rust tool-name bound"); + }); + it("runs the exact fixture validation command and rejects plausible wrong bytes", async () => { + const root = await mkdtemp(join(tmpdir(), "pi-validation-command-")); + const c = piFileContract("0123456789ab-1"); + const validate = () => promisify(execFile)("/bin/sh", ["-c", c.validationCommand], { + cwd: root, timeout: 3_000, maxBuffer: 4096, env: { PATH: dirname(process.execPath) }, + }); + try { + await writeFile(join(root, c.filename), c.after); + expect((await validate()).stdout).toBe(`${c.validationMarker}\n`); + await writeFile(join(root, c.filename), c.after.trimEnd()); + await expect(validate()).rejects.toMatchObject({ code: 1 }); + await rm(join(root, c.filename)); + await expect(validate()).rejects.toMatchObject({ code: 1 }); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it("changes only Pi file prompts and keeps all four question methods", () => { + const cells = runnerMatrix.filter(c => c.suite.id === "extended-harnesses" && c.task.id === "file-edit-validate"); + expect(cells).toHaveLength(6); + for (const cell of cells) { + const prompt = cell.task.buildPrompt("fixture"); + expect(prompt.includes("register_deliverable")).toBe(cell.profile.qualificationCandidate === "pi"); + } + expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi")).toHaveLength(25); + }); + it.each(["local", "daytona"] as const)("seeds once and collects public %s evidence", async environment => { + const root = await mkdtemp(join(tmpdir(), "pi-file-evidence-")); + try { + const seed = await seedPiFile(root, "fixture"), f = environment === "daytona" ? copyback() : fixture(), c = piFileContract("fixture"); + expect(await readFile(join(root, c.filename), "utf8")).toBe(c.before); + await expect(seedPiFile(root, "fixture")).rejects.toThrow(); + await writeFile(join(root, c.filename), c.after); + const get = vi.fn(async (path: string) => { + if (path === "/api/heartbeat-runs/run") return f.run; + if (path === "/api/environment-leases/lease" && "lease" in f) return f.lease; + if (path.endsWith("/attachments")) return f.attachments; + if (path.endsWith("/activity")) return f.activity; + throw new Error("Unexpected route"); + }); + const download = vi.fn(async () => ({ ok: () => true, body: async () => f.downloadedBytes })); + const visible = vi.fn(async () => {}), evidence = vi.fn(async () => {}); + const proof = await collectPiFileEvidence({ ...f, seed, workspace: root, evidence, + api: { get, request: { get: download } } as any, + page: { locator: () => ({ first: () => ({ waitFor: visible }) }) } as any }); + expect(proof.passed).toBe(true); + if (environment === "daytona") { + expect(get).toHaveBeenCalledWith("/api/heartbeat-runs/run"); + expect(get).toHaveBeenCalledWith("/api/environment-leases/lease"); + expect(proof.workspaceProvenance.kind).toBe("product_copyback"); + } + expect(visible).toHaveBeenCalledOnce(); + expect(evidence).toHaveBeenCalledWith(expect.objectContaining({ workspaceBytes: f.workspaceBytes.toString("base64"), downloadedBytes: f.downloadedBytes.toString("base64") })); + expect(download).toHaveBeenCalledWith(`/api/attachments/${f.attachmentId}/content`); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); + +function copyback() { + const f = fixture(); + const reference = { schema: "paperclip.native-workspace-sync/v2", state: "finalized", baselineSha256: "a".repeat(64), + descriptorSha256: "b".repeat(64), finalHostSha256: "c".repeat(64), workspaceId: "workspace", leaseId: "lease", + providerLeaseId: "sandbox", remoteCwd: "/workspace" }; + const run = { ...f.run, contextSnapshot: { executionWorkspaceId: "workspace", paperclipEnvironment: { id: "environment", driver: "sandbox", leaseId: "lease" } }, + runnerProfileJson: { nativeWorkspaceSync: reference, nativeExecutionInput: { binding: { executionWorkspaceId: "workspace" } } } }; + const lease = { id: "lease", companyId: "company", environmentId: "environment", heartbeatRunId: "run", provider: "daytona", providerLeaseId: "sandbox", metadata: { remoteCwd: "/workspace" } }; + return { ...f, run, lease, environment: "daytona" as const }; +} +it("accepts public finalized same-run copy-back attribution without claiming guest observation", () => { + const f = copyback(); + expect(gradePiFileEvidence(f).workspaceProvenance).toMatchObject({ kind: "product_copyback", guestObservation: false, hostSnapshotRecomputed: false }); +}); +it.each([ + ["missing reference", f => { f.run.runnerProfileJson.nativeWorkspaceSync = {} as any; }], + ["prepared only", f => { f.run.runnerProfileJson.nativeWorkspaceSync.state = "prepared"; }], + ["missing baseline", f => { f.run.runnerProfileJson.nativeWorkspaceSync.baselineSha256 = ""; }], + ["invalid final hash", f => { f.run.runnerProfileJson.nativeWorkspaceSync.finalHostSha256 = "wrong"; }], + ["foreign workspace", f => { f.run.contextSnapshot.executionWorkspaceId = "other"; }], + ["contradictory native binding", f => { f.run.runnerProfileJson.nativeExecutionInput.binding.executionWorkspaceId = "other"; }], + ["foreign lease", f => { f.lease.id = "other"; }], + ["foreign run", f => { f.lease.heartbeatRunId = "other"; }], + ["foreign company", f => { f.lease.companyId = "other"; }], + ["foreign environment", f => { f.lease.environmentId = "other"; }], + ["foreign provider lease", f => { f.lease.providerLeaseId = "other"; }], + ["wrong remote root", f => { f.lease.metadata.remoteCwd = "/other"; }], + ["wrong provider", f => { f.lease.provider = "other"; }], +] satisfies Array<[string, (f: ReturnType) => void]>)("rejects %s copy-back provenance", (_label, change) => { + const f = copyback(); change(f); + expect(() => gradePiCopyback(f.run, f.lease, f.companyId, f.environmentId)).toThrow("Pi file evidence:"); +}); diff --git a/tests/runner-e2e/pi-file-evidence.ts b/tests/runner-e2e/pi-file-evidence.ts new file mode 100644 index 00000000000..1a3e96326e2 --- /dev/null +++ b/tests/runner-e2e/pi-file-evidence.ts @@ -0,0 +1,192 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { Page } from "@playwright/test"; +import type { RunnerApi } from "./api.js"; +import { isValidNativePrpEnvelope } from "./native-event-envelope.js"; + +type Row = Record; +const rec = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {}; +const sha = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); +const requireEvidence = (value: unknown, reason: string) => { if (!value) throw new Error(`Pi file evidence: ${reason}`); }; +export function piFileContract(nonce: string) { + if (!/^[A-Za-z0-9-]{1,100}$/u.test(nonce)) throw new Error("Invalid file fixture nonce"); + const filename = `extended-${nonce}.txt`, before = `ready-${nonce}\n`, after = `verified-${nonce}\n`; + const validationMarker = `PI-VALIDATED-${nonce}`; + const validationCommand = `node -e 'if(require("node:fs").readFileSync(${JSON.stringify(filename)},"utf8")!==${JSON.stringify(after)})throw Error("mismatch");console.log(${JSON.stringify(validationMarker)})'`; + if ([...validationCommand].length > 240) throw new Error("Validation command exceeds native Rust tool-name bound"); + return { filename, before, after, sha256: sha(after), byteSize: Buffer.byteLength(after), validationMarker, validationCommand }; +} +export function piFilePrompt(nonce: string) { + const c = piFileContract(nonce); + return [ + `The fixture has already created ${c.filename} in your execution workspace with exactly ${JSON.stringify(c.before)}. Read it, then use your native edit tool once to replace those bytes with exactly ${JSON.stringify(c.after)}. Do not substitute write, bash, or an API for the edit.`, + `Run this exact native bash command, without an added shell prefix or suffix:\n${c.validationCommand}\nIt independently reads the file, fails on incorrect bytes, and prints the validation marker only after success.`, + `Deliver the edited text file as a downloadable attachment using the assigned register_deliverable semantic tool, with filename and workspace-relative contentRef ${c.filename}, contentType text/plain, byteSize ${c.byteSize}, sha256 ${c.sha256}, title Verified text, and a fresh idempotencyKey. Wait for the real registration receipt. Do not upload a different file or claim publication from a disk path.`, + `After successful validation and registration, call paperclip_finish with reportedWorkDisposition done, summary EXTENDED-FILE-${nonce}, the current completion contract revision and satisfied objective criterion, no remaining work, the actual validation command/result, and the registered artifact reference.`, + `Wait for paperclip_finish to succeed, then emit exactly EXTENDED-FILE-${nonce} as your final response. Do not create unrelated files or work.`, + ].join("\n"); +} +export async function seedPiFile(workspace: string, nonce: string) { + const c = piFileContract(nonce); + await writeFile(join(workspace, c.filename), c.before, { flag: "wx" }); + const bytes = await readFile(join(workspace, c.filename)); + requireEvidence(bytes.equals(Buffer.from(c.before)), "seed bytes changed before provider startup"); + return { schema: "paperclip.e2e.pi-file-seed.v1", filename: c.filename, beforeSha256: sha(bytes), byteSize: bytes.length }; +} + +export function gradePiCopyback(run: Row, lease: Row, companyId: string, environmentId: string) { + const context = rec(run.contextSnapshot), environment = rec(context.paperclipEnvironment); + const reference = rec(rec(run.runnerProfileJson).nativeWorkspaceSync); + const binding = rec(rec(rec(run.runnerProfileJson).nativeExecutionInput).binding); + requireEvidence(run.companyId === companyId && reference.schema === "paperclip.native-workspace-sync/v2" + && reference.state === "finalized" && [reference.baselineSha256, reference.descriptorSha256, reference.finalHostSha256] + .every(value => typeof value === "string" && /^[a-f0-9]{64}$/u.test(value)), "finalized production copy-back hashes required"); + requireEvidence(typeof reference.workspaceId === "string" && reference.workspaceId.length > 0 + && reference.workspaceId === context.executionWorkspaceId + && (binding.executionWorkspaceId === undefined || binding.executionWorkspaceId === reference.workspaceId) + && environment.id === environmentId && environment.driver === "sandbox" && environment.leaseId === reference.leaseId, + "copy-back workspace and environment must belong to the actual run"); + requireEvidence(lease.id === reference.leaseId && lease.companyId === companyId && lease.environmentId === environmentId + && lease.heartbeatRunId === run.id && lease.provider === "daytona" && typeof reference.providerLeaseId === "string" + && reference.providerLeaseId.length > 0 && lease.providerLeaseId === reference.providerLeaseId + && typeof reference.remoteCwd === "string" && reference.remoteCwd.startsWith("/") + && reference.remoteCwd === rec(lease.metadata).remoteCwd, "copy-back must bind the exact run and provider lease"); + return { kind: "product_copyback", reference, lease: { id: lease.id, companyId, environmentId, heartbeatRunId: run.id, + provider: lease.provider, providerLeaseId: lease.providerLeaseId, remoteCwd: reference.remoteCwd }, + guestObservation: false, hostSnapshotRecomputed: false }; +} + +/** Only the common public projection is authority here. Native raw arguments, + * full diff blocks and typed command exit codes are deliberately not invented. */ +export function gradePiFileEvidence(input: { + nonce: string; companyId: string; issueId: string; agentId: string; run: Row; + environment: "local" | "daytona"; environmentId: string; lease?: Row; + events: unknown[]; seed: unknown; workspaceBytes: Buffer; attachments: unknown[]; + attachmentId: string; downloadedBytes: Buffer; activity: unknown[]; +}) { + const c = piFileContract(input.nonce), run = input.run, seed = rec(input.seed); + const workspaceProvenance = input.environment === "daytona" + ? gradePiCopyback(run, rec(input.lease), input.companyId, input.environmentId) + : { kind: "independent_local_workspace" }; + requireEvidence(run.id && run.companyId === input.companyId && run.agentId === input.agentId + && run.runtimeMode === "native" && run.status === "succeeded", "actual native run identity"); + requireEvidence(seed.schema === "paperclip.e2e.pi-file-seed.v1" && seed.filename === c.filename + && seed.beforeSha256 === sha(c.before) && seed.byteSize === Buffer.byteLength(c.before), "independent pre-edit seed"); + requireEvidence(input.workspaceBytes.equals(Buffer.from(c.after)), "independent final workspace bytes"); + requireEvidence(input.events.length > 0 && input.events.length <= 20_000, "bounded run events"); + const rows = input.events.map(rec).filter(row => rec(row.payload).prpEvent !== undefined); + const seen = new Set(), seqs = new Set(), sourceSeqs = new Map(); + const canonical = rows.map(row => { + const e = rec(rec(row.payload).prpEvent); + requireEvidence(row.companyId === input.companyId && row.runId === run.id && e.runId === run.id + && isValidNativePrpEnvelope(e, row.protocolSchemaVersion) && row.eventType === e.eventType + && Number.isSafeInteger(row.seq) && row.seq > 0 && !seqs.has(row.seq) + && typeof e.sourceInstanceId === "string" && e.sourceInstanceId.length > 0 + && Number.isSafeInteger(e.sourceSeq) && e.sourceSeq > (sourceSeqs.get(e.sourceInstanceId) ?? 0) + && e.sourceEventId === `${e.sourceInstanceId}:${e.runId}:${e.sourceSeq}` && !seen.has(e.sourceEventId), "foreign, duplicated, or reordered durable evidence"); + seqs.add(row.seq); seen.add(e.sourceEventId); sourceSeqs.set(e.sourceInstanceId, e.sourceSeq); + return { row, e, p: rec(e.payload) }; + }); + const tools = canonical.filter(x => x.e.eventType.startsWith("tool.execution.") && x.p.transport === "builtin"); + const edits = tools.filter(x => x.e.eventType === "tool.execution.completed" && x.p.operation === "edit"); + requireEvidence(edits.length === 1, "one native edit required; final-only writes cannot qualify"); + const edit = edits[0]!; + requireEvidence(edit.p.name === "edit" && edit.p.target === c.filename, "exact native edit and relative target required"); + function lifecycle(end: typeof edit) { + const matches = tools.filter(x => x.p.executionId === end.p.executionId); + const starts = matches.filter(x => x.e.eventType === "tool.execution.started"); + requireEvidence(typeof end.p.executionId === "string" && end.p.executionId.length > 0 + && end.p.schema === "paperclip.tool.execution.v1" && end.p.status === "completed" + && end.e.sourceKind === "runner" && end.e.sourceInstanceId === run.runnerInstanceId + && end.e.normalizedSessionId === run.nativeSessionId && typeof end.e.turnId === "string" && end.e.turnId.length > 0 + && typeof end.e.normalizedSessionId === "string" && end.e.normalizedSessionId.length > 0 + && starts.length === 1 && starts[0]!.row.seq < end.row.seq + && matches.filter(x => x.e.eventType === "tool.execution.completed").length === 1 + && matches.every(x => x.p.schema === end.p.schema && x.p.name === end.p.name && x.p.operation === end.p.operation + && x.p.target === end.p.target && x.e.sourceKind === "runner" && x.e.turnId === end.e.turnId + && x.e.normalizedSessionId === end.e.normalizedSessionId && x.e.sourceInstanceId === end.e.sourceInstanceId + && (x === end || x.p.status === "running")), "incomplete or unsuccessful native tool lifecycle"); + return starts[0]!; + } + lifecycle(edit); + const validations = tools.filter(x => x.e.eventType === "tool.execution.completed" && x.p.operation === "execute" + && x.p.name === c.validationCommand && typeof x.p.output === "string" && x.p.output.includes(c.validationMarker)); + requireEvidence(validations.length === 1, "one observed provider validation execution"); + const validation = validations[0]!, start = lifecycle(validation); + requireEvidence(start.row.seq > edit.row.seq && validation.e.turnId === edit.e.turnId + && validation.e.normalizedSessionId === edit.e.normalizedSessionId + && validation.e.sourceInstanceId === edit.e.sourceInstanceId && validation.p.outputTruncated === false, + "validation must follow the edit in the same native turn"); + const attachments = input.attachments.map(rec).filter(a => a.id === input.attachmentId); + requireEvidence(attachments.length === 1, "one public attachment"); + const a = attachments[0]!; + requireEvidence(a.companyId === input.companyId && a.issueId === input.issueId && a.originatingRunId === run.id + && a.createdByAgentId === input.agentId && a.originalFilename === c.filename && a.contentType === "text/plain" + && a.byteSize === c.byteSize && a.sha256 === c.sha256, "attachment bytes and immutable run attribution"); + requireEvidence(input.downloadedBytes.equals(Buffer.from(c.after)), "actual public download bytes"); + const receipts = Object.values(rec(rec(run.resultJson).semanticToolReceipts)).map(rec) + .filter(r => r.operationId === "register_deliverable" && rec(r.input).filename === c.filename); + requireEvidence(receipts.length === 1, "one authenticated semantic registration"); + const receipt = receipts[0]!, args = rec(receipt.input), result = rec(receipt.result); + requireEvidence(args.contentRef === c.filename && args.contentType === "text/plain" && args.byteSize === c.byteSize + && args.sha256 === c.sha256 && result.disposition === "applied" && result.commandId === `deliverable-prepared:${a.id}` + && Array.isArray(result.entityRefs) && result.entityRefs[0] === a.id, "registered artifact receipt must match downloaded attachment"); + const publications = input.activity.map(rec).filter(row => row.action === "issue.attachment_added" + && rec(row.details).attachmentId === a.id); + requireEvidence(publications.length === 1 && publications[0]!.companyId === input.companyId + && publications[0]!.runId === run.id && publications[0]!.actorId === input.agentId + && publications[0]!.entityId === input.issueId && rec(publications[0]!.details).source === "paperclip_runner_protocol", + "publication activity must belong to this run and task"); + return { + schema: "paperclip.e2e.pi-file-evidence.v1", passed: true, workspaceProvenance, + runId: run.id, turnId: edit.e.turnId, nativeEditExecutionId: edit.p.executionId, + validationExecutionId: validation.p.executionId, attachmentId: a.id, + verification: { kind: "independent_exact_bytes", beforeSha256: sha(c.before), afterSha256: c.sha256, + downloadedSha256: sha(input.downloadedBytes), byteSize: c.byteSize, providerExecutionStatus: validation.p.status, + commandTitle: validation.p.name, nativeFileTarget: edit.p.target, + nativeFileAttribution: "workspace_relative_display_target" }, + diff: { source: "independent_seed_and_workspace_bytes", path: c.filename, + text: `--- ${c.filename}\n+++ ${c.filename}\n@@ -1 +1 @@\n-${c.before.trimEnd()}\n+${c.after.trimEnd()}\n` }, + limits: ["Validation command title is checked exactly; raw arguments and typed exitCode are not projected, so raw invocation/exit code are not attested.", + "Diff is computed from independently checked workspace bytes; it does not qualify native diff presentation or private invocation metadata."], + }; +} + +export async function collectPiFileEvidence(input: { + page: Page; api: RunnerApi; nonce: string; companyId: string; issueId: string; agentId: string; + run: Row; events: unknown[]; seed: unknown; workspace: string; + environment: "local" | "daytona"; environmentId: string; + evidence(data: unknown): Promise; +}) { + const c = piFileContract(input.nonce); + let run = input.run, lease: Row | undefined; + if (input.environment === "daytona") { + run = await input.api.get(`/api/heartbeat-runs/${input.run.id}`); + requireEvidence(run.id === input.run.id, "public copy-back run readback changed identity"); + const leaseId = rec(rec(run.runnerProfileJson).nativeWorkspaceSync).leaseId; + requireEvidence(typeof leaseId === "string" && /^[A-Za-z0-9_-]{1,128}$/u.test(leaseId), "copy-back lease reference missing"); + lease = await input.api.get(`/api/environment-leases/${leaseId}`); + } + const attachments = await input.api.get(`/api/issues/${input.issueId}/attachments`); + const selected = attachments.filter(a => a.originalFilename === c.filename); + requireEvidence(selected.length === 1 && /^[a-f0-9-]{36}$/u.test(selected[0]!.id), "exact published file selection"); + requireEvidence(selected[0]!.byteSize === c.byteSize && selected[0]!.sha256 === c.sha256 + && selected[0]!.contentType === "text/plain", "bounded attachment metadata before download"); + const attachmentId = selected[0]!.id; + const contentPath = `/api/attachments/${attachmentId}/content`; + // Exercise the real task surface and the authenticated public download route. + await input.page.locator(`a[href*="${contentPath}"]`).first().waitFor({ state: "visible", timeout: 15_000 }); + const response = await input.api.request.get(contentPath); + requireEvidence(response.ok(), "public attachment download failed"); + const downloadedBytes = await response.body(); + requireEvidence(downloadedBytes.length <= 4096, "bounded text attachment"); + const workspaceBytes = await readFile(join(input.workspace, c.filename)); + const activity = await input.api.get(`/api/issues/${input.issueId}/activity`); + await input.evidence({ schema: "paperclip.e2e.pi-file-observation.v1", attachments, attachmentId, activity, + workspaceBytes: workspaceBytes.toString("base64"), downloadedBytes: downloadedBytes.toString("base64"), + encoding: "base64", expectedSha256: c.sha256, copybackReference: rec(run.runnerProfileJson).nativeWorkspaceSync, + copybackLease: lease && { id: lease.id, companyId: lease.companyId, environmentId: lease.environmentId, + heartbeatRunId: lease.heartbeatRunId, provider: lease.provider, providerLeaseId: lease.providerLeaseId, remoteCwd: rec(lease.metadata).remoteCwd } }); + return gradePiFileEvidence({ ...input, run, lease, attachments, attachmentId, downloadedBytes, workspaceBytes, activity }); +} diff --git a/tests/runner-e2e/pi-native-cases.test.ts b/tests/runner-e2e/pi-native-cases.test.ts index 1c372652ec1..1df4265fa6f 100644 --- a/tests/runner-e2e/pi-native-cases.test.ts +++ b/tests/runner-e2e/pi-native-cases.test.ts @@ -5,19 +5,21 @@ import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; describe("Pi native Product qualification", () => { - it("selects four local and three remote Pi cases without changing the basic extended matrix", () => { + it("selects five local and four remote Pi cases without changing the basic extended matrix", () => { const suite = runnerSuites.find(row => row.id === "pi-native")!; - expect(suite.manualOnly).toBe(true); expect(suite.expectedMatrixSize).toBe(7); + expect(suite.manualOnly).toBe(true); expect(suite.expectedMatrixSize).toBe(9); const cells = runnerMatrix.filter(row => row.suite.id === suite.id); - expect(cells).toHaveLength(7); + expect(cells).toHaveLength(9); expect(cells.every(row => row.profile.qualificationCandidate === "pi")).toBe(true); expect(cells.filter(row => row.environment.id === "local").map(row => [row.task.id, row.task.expectedRunCount])).toEqual([ ["native-questions", 1], ["agent-files-fresh-run", 2], ["restrictive-denial", 1], ["human-permission-denial", 1], + ["native-pending-controller-restart", 1], ]); expect(cells.filter(row => row.environment.id === "daytona").map(row => [row.task.id, row.task.expectedRunCount])).toEqual([ ["native-questions", 1], ["agent-files-fresh-run", 2], ["human-permission-denial", 1], + ["native-pending-controller-restart", 1], ]); - expect(cells.reduce((sum, row) => sum + row.task.expectedRunCount, 0)).toBe(9); + expect(cells.reduce((sum, row) => sum + row.task.expectedRunCount, 0)).toBe(11); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.suite.id === suite.id)).toBe(false); expect(runnerMatrix.filter(row => row.suite.id === "extended-harnesses")).toHaveLength(30); expect(piNativeTasks[0]!.buildPrompt("fixture")).toContain("paperclip_native_question"); diff --git a/tests/runner-e2e/pi-native-cases.ts b/tests/runner-e2e/pi-native-cases.ts index e83077aca87..8daadeae7c7 100644 --- a/tests/runner-e2e/pi-native-cases.ts +++ b/tests/runner-e2e/pi-native-cases.ts @@ -6,6 +6,7 @@ export const piNativeTasks: readonly RunnerTaskFixture[] = [ ["agent-files-fresh-run", "Agent files save and survive a fresh task", 2], ["restrictive-denial", "Restrictive native write denial prevents a file effect", 1], ["human-permission-denial", "Browser denial prevents a native write through process retirement", 1], + ["native-pending-controller-restart", "Pending native question survives controller restart in the same live run", 1], ].map(([id, label, turns]) => ({ id: String(id), label: String(label), groups: [], workMode: "standard", flow: "pi_native", expectedRunCount: Number(turns), attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 }, @@ -21,6 +22,11 @@ export function piNativeFinish(marker: string): string { } export function piNativePrompt(id: string, nonce: string): string { + if (id === "native-pending-controller-restart") return [ + 'Call paperclip_native_question exactly once with {method:"input",title:"Pi native restart",placeholder:"Answer"}. Wait for the real answer; never guess it or substitute semantic request_human_input.', + "Use native write to save the exact returned tool-result object as JSON to pi-native-restart-answer.json in the task workspace. Do not change or infer its fields. This is an internal verification file, not a deliverable.", + piNativeFinish(`PI-NATIVE-${id}-${nonce}`), + ].join("\n"); if (id === "native-questions") return [ "Use paperclip_native_question exactly four times, sequentially, waiting for each answer. Do not substitute semantic request_human_input.", 'First: {method:"select",title:"Pi native color",options:[{id:"blue",label:"Blue"},{id:"red",label:"Red"}]}.', diff --git a/tests/runner-e2e/pi-native-flow.ts b/tests/runner-e2e/pi-native-flow.ts index eadfda593d0..7bd86e2ffa2 100644 --- a/tests/runner-e2e/pi-native-flow.ts +++ b/tests/runner-e2e/pi-native-flow.ts @@ -12,6 +12,7 @@ import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; import type { RemoteNativeFixture, RemoteNativeSnapshot } from "./remote-native-fixtures.js"; +import { runPiPendingControllerRestart } from "./pi-native-restart-flow.js"; export interface PiRemoteBootstrap { prompt(nonce: string): string; @@ -92,7 +93,17 @@ export async function runPiNativeFlow(input: { return result; } try { - if (execution.task.id === "native-questions") { + if (execution.task.id === "native-pending-controller-restart") { + await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: ["pi-native-restart-answer.json"] }); + checks.push(...await runPiPendingControllerRestart({ + page, companyId: fixtures.company.id, deadlineAt: input.deadlineAt, load, events, + restart: input.restart, settle: () => settle(1), capture: input.capture, evidence: input.evidence, + readProof: async () => { + if (remote) await finishRemote("native-restart-final"); + return JSON.parse(await readWorkspace("pi-native-restart-answer.json")); + }, + })); + } else if (execution.task.id === "native-questions") { const name = `name-${randomBytes(8).toString("hex")}`; const draft = `draft-${randomBytes(8).toString("hex")}\nSecond line`; await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: ["pi-native-answers.json"] }); const seen = new Set(); let runId: string | undefined; diff --git a/tests/runner-e2e/pi-native-restart-flow.test.ts b/tests/runner-e2e/pi-native-restart-flow.test.ts new file mode 100644 index 00000000000..b6e0be51c82 --- /dev/null +++ b/tests/runner-e2e/pi-native-restart-flow.test.ts @@ -0,0 +1,152 @@ +import { expect, it, vi } from "vitest"; +import { gradePiRestartCompletion, observePiRestartPending, runPiPendingControllerRestart } from "./pi-native-restart-flow.js"; +import { validatePrpStructuredRunResult } from "../../packages/paperclip-runner/src/protocol/replay-contract.js"; + +type Row = Record; +vi.mock("@playwright/test", () => ({ expect: (locator: { count(): number }) => ({ + toHaveCount: async (expected: number) => expect(locator.count()).toBe(expected), +}) })); + +function fixture(adapter = "acpx-runtime-sidecar") { + const questionSet = { schema: "paperclip.question_set.v1", questions: [{ id: "answer", header: "Pi native restart", prompt: "Answer", answerMode: "text", required: true }] }; + const request = { schema: "paperclip.runtime_request.v2", requestKind: "runtime", type: "input", status: "pending", requestId: "request", turnId: "turn", itemId: "request", + input: questionSet, origin: { adapter, provider: "pi", method: "elicitation/create" } }; + const wrap = (eventType: string, sourceSeq: number, payload: Row): Row => ({ + companyId: "company", runId: "run", seq: sourceSeq, protocolSchemaVersion: 1, eventType, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", eventType, runId: "run", turnId: "turn", itemId: "request", + normalizedSessionId: "session", sourceInstanceId: "runner", sourceSeq, sourceEventId: `runner:run:${sourceSeq}`, payload } }, + }); + const state = { issue: { id: "issue", companyId: "company", status: "in_progress" }, + runs: [{ id: "run", companyId: "company", nativeIssueId: "issue", runtimeMode: "native", status: "running", nativeSessionId: "session", runnerInstanceId: "runner" } as Row], + interactions: [{ id: "interaction", companyId: "company", issueId: "issue", kind: "ask_user_questions", status: "pending", result: null, + sourceRunId: "run", continuationPolicy: "none", resolverPolicy: "human_only", idempotencyKey: "paperclip-runner-question:run:request", + payload: { runtimeRequestId: "request", questionSet } } as Row] }; + const events = [wrap("runtime_request.created", 1, { request })]; + const control = (eventType: string, sourceSeq: number, payload: Row): Row => { + const row = wrap(eventType, sourceSeq + 4, payload), event = row.payload.prpEvent; + Object.assign(event, { sourceKind: "control_plane", sourceInstanceId: "runner:control", sourceSeq, sourceEventId: `runner:control:run:${sourceSeq}` }); + delete event.itemId; + Object.assign(row, { sourceInstanceId: event.sourceInstanceId, sourceSeq, sourceEventId: event.sourceEventId }); + return row; + }; + function finish(answer = "hidden") { + state.issue.status = "done"; state.runs[0]!.status = "succeeded"; + Object.assign(state.interactions[0]!, { status: "answered", result: { answers: [{ questionId: "answer", optionIds: [], otherText: answer }] } }); + const validated = validatePrpStructuredRunResult({ summary: "Restart question answered", reportedWorkDisposition: "done", + completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], verification: [] }); + if (!validated.ok) throw new Error("Fixture finish result must satisfy the actual production schema"); + const result = validated.result; + events.push(wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", itemId: "request", action: "submit", + response: { schema: "paperclip.question_response.v1", answers: { answer: { text: answer } } } }), wrap("turn.completed", 4, { status: "completed" }), + wrap("run.result.proposed", 3, structuredClone(result)), control("run.result.accepted", 1, { result: structuredClone(result) }), + control("run.terminal", 2, { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" })); + return { status: "answered", value: answer }; + } + return { state, events, request, wrap, control, finish }; +} + +it.each(["acpx-runtime", "acpx-runtime-sidecar"])("accepts mixed runner/control-plane production completion via %s", adapter => { + const f = fixture(adapter), pending = observePiRestartPending(f.state, f.events, "company"), proof = f.finish(); + expect(gradePiRestartCompletion(f.state, f.events, pending, "hidden", proof)).toBe(true); +}); + +it.each([ + ["no native event", (f: ReturnType) => { f.events.length = 0; }], + ["wrong origin", (f: ReturnType) => { f.request.origin.provider = "cursor"; }], + ["semantic origin", (f: ReturnType) => { f.request.origin.adapter = "semantic"; }], + ["internal kind instead of public schema", (f: ReturnType) => { f.request.requestKind = "elicitation"; }], + ["missing native session", (f: ReturnType) => { delete f.state.runs[0]!.nativeSessionId; }], + ["foreign request turn", (f: ReturnType) => { f.request.turnId = "other"; }], + ["foreign company", (f: ReturnType) => { f.state.interactions[0]!.companyId = "other"; }], + ["durable fallback", (f: ReturnType) => { f.state.interactions[0]!.continuationPolicy = "wake_assignee"; }], + ["wrong idempotency identity", (f: ReturnType) => { f.state.interactions[0]!.idempotencyKey = "other"; }], + ["duplicate interaction", (f: ReturnType) => { f.state.interactions.push(structuredClone(f.state.interactions[0]!)); }], + ["duplicate source receipt", (f: ReturnType) => { f.events.push(structuredClone(f.events[0]!)); }], + ["already answered", (f: ReturnType) => { f.finish(); }], +] as const)("rejects %s before restart", (_label, mutate) => { + const f = fixture(); mutate(f); expect(() => observePiRestartPending(f.state, f.events, "company")).toThrow("Pi native restart"); +}); + +it.each([ + ["replacement run", (f: ReturnType) => { f.state.runs[0]!.id = "replacement"; }], + ["additional run", (f: ReturnType) => { f.state.runs.push({ ...f.state.runs[0], id: "replayed" }); }], + ["replacement native session", (f: ReturnType) => { f.state.runs[0]!.nativeSessionId = "replacement"; }], + ["replacement producer", (f: ReturnType) => { f.state.runs[0]!.runnerInstanceId = "replacement"; }], + ["replaced interaction", (f: ReturnType) => { f.state.interactions[0]!.id = "replacement"; }], + ["missing resolution", (f: ReturnType) => { f.events.splice(1, 1); }], + ["duplicate resolution", (f: ReturnType) => { f.events.push(structuredClone(f.events[1]!)); }], + ["replayed creation", (f: ReturnType) => { f.events.push(f.wrap("runtime_request.created", 4, { request: f.request })); }], + ["cancelled callback", (f: ReturnType) => { f.events[1]!.eventType = f.events[1]!.payload.prpEvent.eventType = "runtime_request.cancelled"; }], + ["expired callback", (f: ReturnType) => { f.events[1]!.eventType = f.events[1]!.payload.prpEvent.eventType = "runtime_request.expired"; }], + ["wrong delivered answer", (f: ReturnType) => { f.events[1]!.payload.prpEvent.payload.response.answers.answer.text = "guess"; }], + ["wrong durable answer", (f: ReturnType) => { f.state.interactions[0]!.result.answers[0].otherText = "guess"; }], + ["wrong response turn", (f: ReturnType) => { f.events[1]!.payload.prpEvent.payload.turnId = "other"; }], + ["missing resolution envelope turn", (f: ReturnType) => { delete f.events[1]!.payload.prpEvent.turnId; }], + ["missing resolution item", (f: ReturnType) => { delete f.events[1]!.payload.prpEvent.itemId; }], + ["rewritten canonical question", (f: ReturnType) => { f.request.input.questions[0]!.prompt = "Replacement prompt"; }], + ["new native turn", (f: ReturnType) => { f.events[2]!.payload.prpEvent.turnId = "other"; }], + ["failed terminal payload", (f: ReturnType) => { f.events[2]!.payload.prpEvent.payload.status = "failed"; }], + ["missing terminal", (f: ReturnType) => { f.events.splice(2, 1); }], + ["duplicate terminal", (f: ReturnType) => { f.events.push(f.wrap("turn.completed", 4, {})); }], + ["missing control-plane acceptance", (f: ReturnType) => { f.events.splice(4, 1); }], + ["missing control-plane terminal", (f: ReturnType) => { f.events.pop(); }], + ["foreign control-plane source", (f: ReturnType) => { f.events[4]!.payload.prpEvent.sourceInstanceId = "other:control"; }], + ["foreign control-plane turn", (f: ReturnType) => { f.events[4]!.payload.prpEvent.turnId = "other"; }], + ["missing control-plane turn", (f: ReturnType) => { delete f.events[4]!.payload.prpEvent.turnId; }], + ["foreign control-plane session", (f: ReturnType) => { f.events[4]!.payload.prpEvent.normalizedSessionId = "other"; }], + ["mismatched durable control source", (f: ReturnType) => { f.events[4]!.sourceSeq = 5; }], + ["duplicate control receipt", (f: ReturnType) => { f.events.push(structuredClone(f.events[4]!)); }], + ["control plane impersonates a native turn", (f: ReturnType) => { f.events.push(f.control("turn.completed", 3, { status: "completed" })); }], + ["failed control terminal", (f: ReturnType) => { f.events[5]!.payload.prpEvent.payload.runTerminalState = "failed"; }], + ["unaccepted native result", (f: ReturnType) => { f.events[4]!.payload.prpEvent.payload.result.summary = "Replacement result"; }], + ["malformed accepted result", (f: ReturnType) => { delete f.events[4]!.payload.prpEvent.payload.result.schema; }], + ["runner impersonates acceptance", (f: ReturnType) => { f.events[4]!.payload.prpEvent.sourceKind = "runner"; }], +] as const)("rejects %s despite a correct workspace answer", (_label, mutate) => { + const f = fixture(), pending = observePiRestartPending(f.state, f.events, "company"), proof = f.finish(); + mutate(f); expect(() => gradePiRestartCompletion(f.state, f.events, pending, "hidden", proof)).toThrow(); +}); + +it.each([null, {}, { status: "answered", value: "guess" }, { status: "cancelled", value: "hidden" }, { status: "answered", value: "hidden", invented: true }])( + "rejects missing or wrong independent file proof %j", proof => { + const f = fixture(), pending = observePiRestartPending(f.state, f.events, "company"); f.finish(); + expect(() => gradePiRestartCompletion(f.state, f.events, pending, "hidden", proof)).toThrow(); + }, +); + +async function flow(failure?: "replaced" | "wrong-file" | "missing-resolution") { + const f = fixture(), evidence = new Map(), checkpoints: string[] = []; + let answer = "", restarts = 0, submissions = 0, proof: unknown, resolvePost: ((value: unknown) => void) | undefined; + let postPredicate: ((value: any) => boolean) | undefined; + const composer: any = { count: () => 1, filter: () => composer, locator: () => composer, first: () => composer, + fill: async (value: string) => { expect(restarts).toBe(1); answer = value; } }; + const button: any = { count: () => 1, filter: () => button, click: async () => { + expect(restarts).toBe(1); expect(answer).toMatch(/^PI-RESTART-[a-f0-9]{32}$/); submissions++; + const request = { url: () => "http://fixture/api/issues/issue/interactions/interaction/respond", method: () => "POST", + postDataJSON: () => ({ answers: [{ questionId: "answer", optionIds: [], otherText: answer }] }) }; + expect(postPredicate!(request)).toBe(true); resolvePost!(request); proof = f.finish(answer); + if (failure === "missing-resolution") f.events.splice(1, 1); + } }; + const page: any = { getByTestId: () => composer, getByRole: () => button, reload: async () => { checkpoints.push("reload"); }, + waitForRequest: (predicate: (value: any) => boolean) => { postPredicate = predicate; return new Promise(resolve => { resolvePost = resolve; }); } }; + const result = runPiPendingControllerRestart({ page, companyId: "company", deadlineAt: Date.now() + 1000, + load: async () => structuredClone(f.state), events: async () => structuredClone(f.events), + restart: async () => { + expect(submissions).toBe(0); expect(f.state.runs[0]!.status).toBe("running"); expect(f.state.interactions[0]!.status).toBe("pending"); + expect(answer).toBe(""); restarts++; checkpoints.push("restart"); + if (failure === "replaced") f.state.interactions[0]!.id = "replacement"; + }, + settle: async () => structuredClone(f.state), readProof: async () => failure === "wrong-file" ? { status: "answered", value: "guess" } : proof, + capture: async id => { checkpoints.push(id); }, evidence: async (name, data) => { evidence.set(name, structuredClone(data)); }, + }); + if (failure) await expect(result).rejects.toThrow("Pi native restart"); + else expect((await result).every(check => check.passed)).toBe(true); + expect(restarts).toBe(1); expect(submissions).toBe(failure === "replaced" ? 0 : 1); + expect(checkpoints.slice(0, 3)).toEqual(["pi-restart-pending", "restart", "reload"]); + expect(evidence.has("pi-native-restart-before.json")).toBe(true); + expect(evidence.has("pi-native-restart-after.json")).toBe(true); + expect(evidence.has("pi-native-restart-checks.json")).toBe(true); + expect(JSON.stringify(evidence.get("pi-native-restart-before.json"))).not.toContain("PI-RESTART-"); + if (failure !== "replaced") expect(evidence.has("pi-native-restart-final.json")).toBe(true); +} +it("restarts while unanswered, reloads, then submits hidden text through the exact browser route", () => flow()); +it.each(["replaced", "wrong-file", "missing-resolution"] as const)("retains evidence and fails whole flow on %s", failure => flow(failure)); diff --git a/tests/runner-e2e/pi-native-restart-flow.ts b/tests/runner-e2e/pi-native-restart-flow.ts new file mode 100644 index 00000000000..a590549e3c4 --- /dev/null +++ b/tests/runner-e2e/pi-native-restart-flow.ts @@ -0,0 +1,158 @@ +import { createHash, randomBytes } from "node:crypto"; +import { expect, type Page } from "@playwright/test"; +import { canonicalJson } from "../../packages/shared/src/portability-hash.js"; +import { validatePrpStructuredRunResult } from "../../packages/paperclip-runner/src/protocol/replay-contract.js"; +import { pollUntil } from "./api.js"; +import { hasAcpxNativeOrigin } from "./acpx-native-origin.js"; +import { isValidNativePrpEnvelope } from "./native-event-envelope.js"; + +type Row = Record; +type State = { issue: Row; runs: Row[]; interactions: Row[] }; +type Check = { id: string; passed: boolean; detail: string }; +const rec = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {}; +const id = (value: unknown): value is string => typeof value === "string" && value.length > 0 && value.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(value) && !value.includes("[REDACTED]"); +const digest = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex"); +const closures = new Set(["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"]); +const terminals = new Set(["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]); +function requireProof(value: unknown, reason: string): asserts value { if (!value) throw new Error(`Pi native restart: ${reason}`); } + +export interface PiRestartPending { + companyId: string; issueId: string; runId: string; interactionId: string; requestId: string; + turnId: string; itemId: string; nativeSessionId: string; sourceInstanceId: string; questionId: string; + createdSourceSeq: number; createdRowSha256: string; interactionBindingSha256: string; +} + +function interactionBinding(card: Row) { + return { id: card.id, companyId: card.companyId, issueId: card.issueId, kind: card.kind, + sourceRunId: card.sourceRunId, idempotencyKey: card.idempotencyKey, + continuationPolicy: card.continuationPolicy, resolverPolicy: card.resolverPolicy, payload: card.payload }; +} + +function inspect(state: State, events: readonly Row[], companyId: string) { + requireProof(id(companyId) && id(state.issue.id) && state.issue.companyId === companyId && state.runs.length === 1 && state.interactions.length === 1, + "one company-scoped task, run and question required"); + const run = state.runs[0]!, card = state.interactions[0]!; + requireProof(id(run.id) && run.companyId === companyId && run.nativeIssueId === state.issue.id && run.runtimeMode === "native" + && id(run.nativeSessionId) && id(run.runnerInstanceId), "native run/session identity missing"); + requireProof(id(card.id) && card.companyId === companyId && card.issueId === state.issue.id && card.kind === "ask_user_questions" + && card.sourceRunId === run.id && card.continuationPolicy === "none" && card.resolverPolicy === "human_only", "question is not the native callback"); + requireProof(Array.isArray(events) && events.length > 0 && events.length <= 20_000, "bounded durable events required"); + const rows = events.filter(row => rec(row.payload).prpEvent !== undefined).map(row => ({ row, event: rec(rec(row.payload).prpEvent) })).sort((a, b) => a.row.seq - b.row.seq); + const seenRows = new Set(), seenSources = new Set(), cursors = new Map(); + for (const { row, event } of rows) { + requireProof(row.companyId === companyId && row.runId === run.id && isValidNativePrpEnvelope(event, row.protocolSchemaVersion) + && event.runId === run.id && event.eventType === row.eventType && ["runner", "control_plane"].includes(event.sourceKind) + && Number.isSafeInteger(row.seq) && row.seq > 0 && !seenRows.has(row.seq) + && id(event.sourceInstanceId) && Number.isSafeInteger(event.sourceSeq) && event.sourceSeq > (cursors.get(event.sourceInstanceId) ?? 0) + && event.sourceEventId === `${event.sourceInstanceId}:${run.id}:${event.sourceSeq}` && !seenSources.has(event.sourceEventId), + "foreign, duplicate or reordered durable event"); + // The production finalizer owns this separate, stable two-event stream. + // It cannot impersonate a native request/turn, nor can a runner accept itself. + if (event.sourceKind === "control_plane") { + requireProof(event.sourceInstanceId === `${run.runnerInstanceId}:control` && event.normalizedSessionId === run.nativeSessionId + && row.sourceInstanceId === event.sourceInstanceId && row.sourceSeq === event.sourceSeq && row.sourceEventId === event.sourceEventId + && ((event.eventType === "run.result.accepted" && event.sourceSeq === 1) || (event.eventType === "run.terminal" && event.sourceSeq === 2)), + "foreign or unexpected control-plane completion receipt"); + } else requireProof(!["run.result.accepted", "run.terminal"].includes(event.eventType), "runner cannot claim control-plane completion"); + seenRows.add(row.seq); seenSources.add(event.sourceEventId); cursors.set(event.sourceInstanceId, event.sourceSeq); + } + const created = rows.filter(x => x.event.eventType === "runtime_request.created"); + requireProof(created.length === 1, "exactly one native request creation required"); + const opening = created[0]!, request = rec(opening.event.payload?.request), questionSet = rec(request.input); + const question = Array.isArray(questionSet.questions) && questionSet.questions.length === 1 ? rec(questionSet.questions[0]) : {}; + requireProof(request.schema === "paperclip.runtime_request.v2" && request.requestKind === "runtime" && request.type === "input" && request.status === "pending" + && id(request.requestId) && id(request.itemId) && id(request.turnId) && request.turnId === opening.event.turnId && request.itemId === opening.event.itemId + && hasAcpxNativeOrigin(request.origin, "pi", "elicitation/create") && questionSet.schema === "paperclip.question_set.v1" + && id(question.id) && question.answerMode === "text" && question.header === "Pi native restart" + && card.payload?.runtimeRequestId === request.requestId && digest(card.payload.questionSet) === digest(questionSet) + && card.idempotencyKey === `paperclip-runner-question:${run.id}:${request.requestId}`, "canonical Pi input request identity missing"); + requireProof(rows.filter(x => x.event.turnId != null || x.event.eventType.startsWith("turn.") || x.event.eventType.startsWith("runtime_request.")).every(({ event }) => event.turnId === request.turnId + && event.normalizedSessionId === run.nativeSessionId + && event.sourceInstanceId === (event.sourceKind === "runner" ? run.runnerInstanceId : `${run.runnerInstanceId}:control`)), "turn, native session or producer was replaced"); + requireProof(rows.filter(x => x.event.sourceKind === "control_plane").every(x => x.event.turnId === request.turnId), "control-plane result belongs to another turn"); + const binding: PiRestartPending = { companyId, issueId: state.issue.id, runId: run.id, interactionId: card.id, requestId: request.requestId, + turnId: request.turnId, itemId: request.itemId, nativeSessionId: run.nativeSessionId, sourceInstanceId: run.runnerInstanceId, questionId: question.id, + createdSourceSeq: opening.event.sourceSeq, createdRowSha256: digest(opening.row), interactionBindingSha256: digest(interactionBinding(card)) }; + return { run, card, rows, binding }; +} + +/** Grade public state and PRP receipts; assistant prose is never restart proof. */ +export function observePiRestartPending(state: State, events: readonly Row[], companyId: string): PiRestartPending { + const { run, card, rows, binding } = inspect(state, events, companyId); + requireProof(state.issue.status === "in_progress" && run.status === "running" && card.status === "pending" && card.result == null + && run.resultJson?.nativeCancellation == null && run.resultJson?.startupCancellation == null + && !rows.some(x => closures.has(x.event.eventType) || terminals.has(x.event.eventType) || x.event.sourceKind === "control_plane"), "question is no longer unanswered in a live run"); + return binding; +} + +export function gradePiRestartCompletion(state: State, events: readonly Row[], pending: PiRestartPending, answer: string, proof: unknown): boolean { + const { run, card, rows, binding } = inspect(state, events, pending.companyId); + requireProof(digest(binding) === digest(pending), "original request/run/source identity changed"); + const resolved = rows.filter(x => closures.has(x.event.eventType)), terminal = rows.filter(x => terminals.has(x.event.eventType)); + requireProof(resolved.length === 1 && resolved[0]!.event.eventType === "runtime_request.resolved", "exactly one durable native resolution required"); + const delivered = resolved[0]!.event, response = rec(delivered.payload); + requireProof(response.requestId === pending.requestId && response.turnId === pending.turnId && response.itemId === pending.itemId + && response.action === "submit" && delivered.itemId === pending.itemId && delivered.sourceSeq > pending.createdSourceSeq + && digest(response.response) === digest({ schema: "paperclip.question_response.v1", answers: { [pending.questionId]: { text: answer } } }), + "hidden browser answer was not delivered to the original native request"); + requireProof(terminal.length === 1 && terminal[0]!.event.eventType === "turn.completed" && terminal[0]!.event.payload?.status === "completed" + && terminal[0]!.event.payload?.error == null && terminal[0]!.event.sourceSeq > delivered.sourceSeq + && state.issue.status === "done" && run.status === "succeeded" && run.resultJson?.nativeCancellation == null && run.resultJson?.startupCancellation == null, + "original native turn did not finish exactly once"); + const proposed = rows.filter(x => x.event.eventType === "run.result.proposed"), accepted = rows.filter(x => x.event.eventType === "run.result.accepted"), runTerminal = rows.filter(x => x.event.eventType === "run.terminal"); + requireProof(proposed.length === 1 && accepted.length === 1 && runTerminal.length === 1, "unique proposed and control-plane accepted completion required"); + const result = rec(accepted[0]!.event.payload?.result), validation = validatePrpStructuredRunResult(result); + requireProof(validation.ok && digest(validation.result) === digest(result) && result.reportedWorkDisposition === "done" + && digest(proposed[0]!.event.payload) === digest(result) + && proposed[0]!.row.seq > resolved[0]!.row.seq && proposed[0]!.row.seq < terminal[0]!.row.seq + && accepted[0]!.row.seq > terminal[0]!.row.seq && runTerminal[0]!.row.seq > accepted[0]!.row.seq + && digest(runTerminal[0]!.event.payload) === digest({ schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }), + "control-plane completion does not accept the original successful native result"); + requireProof(card.status === "answered" && Array.isArray(card.result?.answers) && card.result.answers.length === 1 + && card.result.answers[0].questionId === pending.questionId && card.result.answers[0].otherText === answer, "durable interaction answer missing or wrong"); + requireProof(digest(proof) === digest({ status: "answered", value: answer }), "independent workspace proof is missing or wrong"); + return true; +} + +export async function runPiPendingControllerRestart(input: { + page: Page; companyId: string; deadlineAt: number; load(): Promise; events(runId: string): Promise; + restart(): Promise; settle(): Promise; readProof(): Promise; + capture(id: string, label: string, file: string): Promise; evidence(name: string, data: unknown): Promise; +}): Promise { + const checks: Check[] = [], answer = `PI-RESTART-${randomBytes(16).toString("hex")}`; + const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); requireProof(passed, detail); }; + const snapshot = async (name: string) => { + const state = await input.load(), events = state.runs.length === 1 ? await input.events(state.runs[0]!.id) : []; + await input.evidence(name, { ...state, events }); return { state, events }; + }; + try { + await pollUntil({ label: "Pi pending native restart question", deadlineAt: input.deadlineAt, load: input.load, + accept: state => state.interactions.some(card => card.status === "pending"), + reject: state => state.runs.length > 1 || state.runs.some(run => ["failed", "cancelled", "timed_out", "succeeded"].includes(run.status)) ? "Pi ended or created another run before the pending restart boundary" : undefined }); + const before = await snapshot("pi-native-restart-before.json"), pending = observePiRestartPending(before.state, before.events, input.companyId); + check("restart-pending-native-identity", true, "One unanswered native Pi callback belongs to the original running turn and session"); + const composer = () => input.page.getByTestId("question-text-answer-composer").filter({ visible: true }); + await expect(composer()).toHaveCount(1); + await input.capture("pi-restart-pending", "Pi native question before controller restart", "pi-restart-pending.png"); + await input.restart(); + await input.page.reload(); + const after = await snapshot("pi-native-restart-after.json"), resumed = observePiRestartPending(after.state, after.events, input.companyId); + check("restart-same-pending-native-request", digest(resumed) === digest(pending), "Controller restart retained the same unanswered request, run, turn, native session and producer"); + await expect(composer()).toHaveCount(1); + await input.capture("pi-restart-reconnected", "Same Pi question after controller restart", "pi-restart-reconnected.png"); + await composer().locator('[contenteditable="true"],textarea').first().fill(answer); + const route = `/api/issues/${pending.issueId}/interactions/${pending.interactionId}/respond`; + const submitted = input.page.waitForRequest(request => new URL(request.url()).pathname === route && request.method() === "POST", + { timeout: Math.max(1, input.deadlineAt - Date.now()) }); + const button = input.page.getByRole("button", { name: after.state.interactions[0]!.payload.questionSet.submitLabel ?? "Submit answers", exact: true }).filter({ visible: true }); + await expect(button).toHaveCount(1); await button.click(); + const posted = (await submitted).postDataJSON(); + check("restart-exact-browser-answer", Array.isArray(posted?.answers) && posted.answers.length === 1 + && posted.answers[0].questionId === pending.questionId && posted.answers[0].otherText === answer, "Actual browser POST answered the retained question with previously undisclosed text"); + const final = await input.settle(), events = await input.events(pending.runId), proof = await input.readProof(); + await input.evidence("pi-native-restart-final.json", { ...final, events, pending, posted, proof }); + check("restart-native-answer-delivered-once", gradePiRestartCompletion(final, events, pending, answer, proof), + "One durable answer and one successful original native turn delivered the exact typed result after restart"); + return checks; + } finally { await input.evidence("pi-native-restart-checks.json", checks); } +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index e3b003ac2fe..cf284fa310b 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -5,6 +5,7 @@ import { createRemoteNativeBootstrap, createRemoteFixtureClient } from "./remote import { runCleanupWithObservers, verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js"; import { runCopilotProtectionFlow } from "./copilot-protection-flow.js"; import { runPiNativeFlow } from "./pi-native-flow.js"; +import { seedPiFile, collectPiFileEvidence } from "./pi-file-evidence.js"; import { warmManagedFileEvidence } from "./warm-managed-files.js"; import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js"; import { runsCompletionUpdateProbe, completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js"; @@ -574,6 +575,7 @@ for (const execution of executions) { let runtimeLeases: EnvironmentLeaseRecord[] = []; let matcherResults: MatcherResult[] = []; let downloadedResponseProof: Awaited> | undefined; + let piFileSeed: Awaited> | undefined; const completionQuality: CompletionQualityRecord[] = []; const completionEvidence = async (name: string, data: unknown) => { await writeSanitizedJson(snapshotsDir, name, data, secrets); @@ -845,6 +847,11 @@ for (const execution of executions) { }); expect(experimental.enableNativeRunner).toBe(true); + if (execution.suite.id === "extended-harnesses" && execution.profile.qualificationCandidate === "pi" && execution.task.id === "file-edit-validate") { + piFileSeed = await seedPiFile(workspacePath, nonce); + await writeSanitizedJson(snapshotsDir, "pi-file-seed.json", piFileSeed, secrets); + } + if (execution.suite.id === "daytona-git-streaming") { await setupGitStreamingWorkspace(workspacePath); } @@ -2136,6 +2143,15 @@ for (const execution of executions) { await writeSanitizedJson(snapshotsDir, "api-response-pagination.json", paging, secrets); if (!paging.passed) invariantFailures.push(`Bounded response paging was not proven: ${paging.failure}`); } + if (piFileSeed) { + const proof = await collectPiFileEvidence({ page, api, nonce, companyId: fixtures.company.id, + issueId: issue.id, agentId: fixtures.agent.id, run: finalRun, + events: runEventsByRun.find(entry => entry.runId === finalRun.id)?.events ?? [], + seed: piFileSeed, workspace: workspacePath, + environment: execution.environment.id === "daytona" ? "daytona" : "local", environmentId: fixtures.environment.id, + evidence: data => writeSanitizedJson(snapshotsDir, "pi-file-observation.json", data, secrets) }); + await writeSanitizedJson(snapshotsDir, "pi-file-evidence.json", proof, secrets); + } if (execution.suite.id === "task-titles") { const titleEvidence = gradeTaskTitle({ initial: titleCreation?.issue, submitted: titleCreation?.submitted, diff --git a/tests/runner-e2e/server-stop.test.ts b/tests/runner-e2e/server-stop.test.ts index af7e1373643..8cf57bcdc84 100644 --- a/tests/runner-e2e/server-stop.test.ts +++ b/tests/runner-e2e/server-stop.test.ts @@ -46,6 +46,10 @@ function stopper(gracefulTimeoutMs = 2_000) { }); return { stop, logs, errors }; } +function expectSuccessfulWrapperReceipt(wrapper: ChildProcess, messages: unknown[], diagnostics = "") { + expect([wrapper.exitCode, wrapper.signalCode], diagnostics).toEqual([0, null]); + expect(messages, diagnostics).toContainEqual({ exitCode: 0, signalCode: null }); +} afterEach(async () => { for (const child of children.splice(0)) { if (child.exitCode !== null || child.signalCode !== null) continue; @@ -153,15 +157,19 @@ it.skipIf(process.platform === "win32")("lets launcher cancellation join wrapper // during the async close even though the helper sends only one signal. const messages: unknown[] = []; wrapper.on("message", message => messages.push(message)); - const exited = once(wrapper, "exit"); + // Process exit is not IPC completion. Register both barriers before + // cancellation so the final worker receipt is drained before asserting it. + const closed = once(wrapper, "close"); + const disconnected = once(wrapper, "disconnect"); await owner.observe(); const cancellation = stopOwnedProcessTree(wrapper, owner, 2_000, 2_000); await new Promise(resolve => setTimeout(resolve, 250)); if (wrapper.connected) wrapper.send("finish"); - await exited; + await Promise.all([closed, disconnected]); await cancellation; - expect(messages, wrapperErrors).toContainEqual({ message: "close-complete", pid: workerPid }); - expect(messages).toContainEqual({ exitCode: 0, signalCode: null }); + const diagnostics = `${wrapperErrors}\nwrapper exit=${wrapper.exitCode} signal=${wrapper.signalCode} connected=${wrapper.connected}`; + expect(messages, diagnostics).toContainEqual({ message: "close-complete", pid: workerPid }); + expectSuccessfulWrapperReceipt(wrapper, messages, diagnostics); expect(() => process.kill(workerPid!, 0)).toThrow(); } finally { owner.stopObserving(); @@ -169,6 +177,21 @@ it.skipIf(process.platform === "win32")("lets launcher cancellation join wrapper } }); +it.each(["failed", "missing"] as const)("rejects a %s worker receipt even when the wrapper closes successfully", async receipt => { + const { entry } = await fixture(receipt === "failed" + ? "process.send({ exitCode: 23, signalCode: null }, () => process.exit(0));" + : "process.exit(0);"); + const wrapper = start(entry); + const messages: unknown[] = []; + wrapper.on("message", message => messages.push(message)); + const closed = once(wrapper, "close"); + const disconnected = once(wrapper, "disconnect"); + await Promise.all([closed, disconnected]); + expect([wrapper.exitCode, wrapper.signalCode]).toEqual([0, null]); + expect(messages).toEqual(receipt === "failed" ? [{ exitCode: 23, signalCode: null }] : []); + expect(() => expectSuccessfulWrapperReceipt(wrapper, messages)).toThrow(); +}); + it.skipIf(process.platform === "win32")("escalates the owned server group so a hung descendant cannot remain", async () => { const { entry } = await fixture(` @@ -230,6 +253,8 @@ it.skipIf(process.platform === "win32")("recovers a failed cleanup without repea const child = start(entry); await once(child, "message"); let failInspection = false; + let markInspectionFailed!: () => void; + const inspectionFailed = new Promise(resolve => { markInspectionFailed = resolve; }); let owner: ReturnType | undefined; const messages: unknown[] = []; child.on("message", message => { @@ -242,14 +267,20 @@ it.skipIf(process.platform === "win32")("recovers a failed cleanup without repea createOwner: candidate => { owner = createProcessTreeOwner(candidate); return { ...owner, observe: async () => { - if (failInspection) { failInspection = false; throw new Error("transient inspection failure"); } + if (failInspection) { + failInspection = false; + markInspectionFailed(); + throw new Error("transient inspection failure"); + } await owner!.observe(); } }; }, }); try { const failed = expect(stop(child)).rejects.toThrow("transient inspection failure"); - await new Promise(resolve => setTimeout(resolve, 100)); + // Hold the child until the one-shot failure is actually injected. A fixed + // delay can let it exit during the preceding successful inspection. + await inspectionFailed; if (child.connected) child.send("finish"); await failed; const recovery = stop(child);