mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
test: strengthen Pi editing and pending native recovery qualification
This commit is contained in:
1 parent
5eba61ece9
commit
5cd6d3d523
15 files changed
+852
-32
No files matched your search
+4
-2
@@ -35,8 +35,10 @@ Pi's explicit-only `pi-controls` Product suite tests Stop while native permissio
|
||||
is unanswered and browser-originated same-turn steering, locally and on Daytona.
|
||||
Its [fixture contract](../tests/runner-e2e/FIXTURES.md#pi-active-controls) separates
|
||||
control acknowledgment, actual message consumption and owned process retirement.
|
||||
The four cells add coverage without changing the existing 19 Pi cells; they
|
||||
remain unqualified until measured on the exact candidate runtime.
|
||||
The current Pi matrix has 25 explicit cells (13 local and 12 Daytona), including
|
||||
these four controls and pending-native-question controller restart. File editing
|
||||
also requires native edit/validation events and a registered artifact download.
|
||||
All remain unqualified until measured on the exact candidate runtime and harness.
|
||||
|
||||
Use **Runner Evals** for a runner protocol, adapter, transport, native session,
|
||||
tool grant, or one-turn provider qualification question. The workflow checks
|
||||
|
||||
@@ -286,7 +286,7 @@ is explicit-only. No private control-plane hooks or direct database writes are u
|
||||
|
||||
## Pi native boundaries
|
||||
|
||||
The explicit-only `pi-native` suite has four local and three Daytona candidate
|
||||
The explicit-only `pi-native` suite has five local and four Daytona candidate
|
||||
cells, with no automatic retries. The remote suite excludes automatic deny-all
|
||||
because its initial native file read is itself denied. `native-questions` answers the real runner-owned Pi select, confirm, input
|
||||
and editor tool through four durable browser cards, reloading before every answer.
|
||||
@@ -304,12 +304,69 @@ is reconnect evidence only; these cases do not establish provider-death recovery
|
||||
They use public product APIs, real browser answers, and ordinary isolated files;
|
||||
no database writes, private hooks, or fabricated provider results are allowed.
|
||||
|
||||
### Pending native controller restart
|
||||
|
||||
`native-pending-controller-restart` restarts the public controller while one Pi
|
||||
input callback remains unanswered. It requires the same durable interaction,
|
||||
request, live run, native session, turn and producer before and after restart.
|
||||
Only then does the browser submit previously undisclosed text. One durable
|
||||
resolution, one original successful turn and independently read exact workspace
|
||||
JSON prove delivery. A replacement run, replay, cancellation, expiry, rewritten
|
||||
request or merely reloaded browser cannot pass. Full states and PRP identities
|
||||
stay in private snapshots under the existing publication allowlist.
|
||||
|
||||
### Pi file editing and registered artifacts
|
||||
|
||||
Pi's `extended-harnesses/file-edit-validate` seeds exact bytes before startup and
|
||||
requires one native edit lifecycle followed by a successful native bash execution
|
||||
with the exact nonce-bound byte-validation command as its projected title and a
|
||||
validation marker. A marker-only echo cannot pass. Independent final bytes must
|
||||
match the fixture. The
|
||||
real `register_deliverable` receipt, attachment metadata, publication activity,
|
||||
visible task attachment and authenticated public download must all agree on the
|
||||
file's bytes, hash, company, issue, agent and originating run. A file on disk or a
|
||||
model completion claim cannot substitute for publication. Daytona additionally
|
||||
requires the public run's finalized `nativeWorkspaceSync` descriptor, baseline and
|
||||
final-host hashes, bound workspace and public environment lease to match this
|
||||
run/company/environment/provider lease and remote root. The checked host bytes
|
||||
come through production stage-in/copy-back; this is explicitly `product_copyback`
|
||||
provenance, not a sealed guest observation or an independently recomputed whole
|
||||
host-workspace snapshot. Downloaded artifact bytes remain independently checked.
|
||||
|
||||
Private `pi-file-seed.json`, `pi-file-observation.json` and `pi-file-evidence.json`
|
||||
retain the seed, downloaded/workspace bytes, public publication receipts, checked hashes,
|
||||
correlated tool identities and a before/after diff computed from independently
|
||||
checked workspace bytes. This is not native diff presentation. The current
|
||||
common tool projection omits raw arguments and reports typed `exitCode: null`;
|
||||
the oracle checks the command title and completed lifecycle, without claiming raw
|
||||
invocation arguments or a typed exit code. The sidecar converts Pi absolute file
|
||||
locations into bounded workspace-relative display targets; the oracle requires
|
||||
the exact relative filename, but that display value is not file-access authority.
|
||||
Restoring raw arguments, typed exit code and native diff presentation remains a
|
||||
capability follow-up.
|
||||
|
||||
Wrong or missing lifecycle, byte, registration, download or recovery evidence is
|
||||
a candidate failure; transport/infrastructure failures retain the existing
|
||||
harness classification. Positive and plausible-wrong/missing-evidence unit
|
||||
calibrations do not count as paid qualification. All automatic retries stay zero.
|
||||
|
||||
Provider-death coverage remains bounded: the existing patched-ACP package test
|
||||
`provider exit fails promptly and missing owned extension fails admission` runs
|
||||
an actual wrapper child against a deterministic fake Pi child. The bridge test
|
||||
`expires pending requests and discards a late answer exactly once` and driver
|
||||
abort/event-stream tests separately cover pending cleanup with mocked transport.
|
||||
They do not establish a single actual native-provider-death to durable browser
|
||||
request-expiry chain. No local or Daytona paid death case is claimed, and no
|
||||
arbitrary process-kill API is introduced by these fixtures.
|
||||
|
||||
## Pi active controls
|
||||
|
||||
The explicit-only `pi-controls` suite adds `pending-permission-stop` and
|
||||
`same-turn-steering` on local and Daytona, each with one provider run, a
|
||||
120-second active-turn timeout and a 300-second attempt budget. The existing
|
||||
19 Pi cells and their suite fingerprints are unchanged. Catalog presence and
|
||||
120-second active-turn timeout and a 300-second attempt budget. These four
|
||||
control cases retain their behavior; the current matrix totals 25 Pi cells.
|
||||
Pi 1/profile 11 and coverage revisions intentionally change the affected suite
|
||||
fingerprints, so older qualification receipts cannot be reused. Catalog presence and
|
||||
deterministic calibration do not constitute paid qualification.
|
||||
|
||||
Both cases create a task through the browser and select `approve-reads` and
|
||||
|
||||
@@ -39,7 +39,9 @@ native-write evidence, caller-owned control acknowledgments, independent
|
||||
no-effect/retirement proof and the actual production task UI. Daytona also
|
||||
requires the existing immutable image and executable digests. These cells do
|
||||
not establish native follow-up queue persistence or provider-death recovery.
|
||||
They do not change the existing 19 Pi cells or promote Pi's pending profile.
|
||||
The current Pi matrix has 25 explicit cells (13 local and 12 Daytona), including
|
||||
these four controls and the pending-native-question restart cases. Pi's profile
|
||||
remains pending; catalog presence is not live qualification.
|
||||
|
||||
## Provider-free browser bootstrap regression
|
||||
|
||||
@@ -1435,8 +1437,9 @@ mock control plane. Neither suite substitutes for the other.
|
||||
|
||||
### Pi native Product fixtures
|
||||
|
||||
The separate `pi-native` suite defines seven explicit Pi cells: native questions,
|
||||
agent-file persistence and browser permission denial on local and Daytona, plus
|
||||
The separate `pi-native` suite defines nine explicit Pi cells: native questions,
|
||||
agent-file persistence, browser permission denial and controller restart with an
|
||||
unanswered native question on local and Daytona, plus
|
||||
a local-only automatic `restrictive-denial` case. The latter cannot use the remote
|
||||
setup read under `deny-all`, so it is explicitly excluded there. It uses the
|
||||
pinned candidate profile and exact OpenRouter model. It is excluded from `--all`,
|
||||
|
||||
@@ -136,10 +136,10 @@ describe("runner E2E catalog", () => {
|
||||
expect(localIntegrityTasks).toHaveLength(2);
|
||||
expect(openRouterBreadthTasks).toHaveLength(3);
|
||||
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
|
||||
4, 8, 7, 4, 4, 6, 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1,
|
||||
4, 8, 9, 4, 4, 6, 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1,
|
||||
]);
|
||||
expect(validateRunnerCatalog()).toHaveLength(466);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(466);
|
||||
expect(validateRunnerCatalog()).toHaveLength(468);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(468);
|
||||
expect(
|
||||
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
|
||||
).toHaveLength(48);
|
||||
|
||||
@@ -3,6 +3,7 @@ import { piControlTasks } from "./pi-controls-cases.js";
|
||||
import { cursorNativeTasks } from "./cursor-native-cases.js";
|
||||
import { copilotProtectionTasks } from "./copilot-protection-tasks.js";
|
||||
import { piNativeTasks } from "./pi-native-cases.js";
|
||||
import { piFilePrompt } from "./pi-file-evidence.js";
|
||||
import { chatConfirmationTasks } from "./chat-cases.js";
|
||||
import { instructionPersistenceTask } from "./instruction-persistence.js";
|
||||
import { apiResponseReadingTask } from "./api-response-reading.js";
|
||||
@@ -1102,9 +1103,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
id: "pi-native", label: "Pi native boundaries", manualOnly: true,
|
||||
description: "Pi native forms, registered agent files and human permission denial on local and Daytona execution; automatic deny-all remains local-only.",
|
||||
groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"),
|
||||
environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 7,
|
||||
environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 9,
|
||||
excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial"],
|
||||
definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer" },
|
||||
definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request" },
|
||||
},
|
||||
{
|
||||
id: "native-active-stop", label: "Stop an unanswered native permission", manualOnly: true,
|
||||
@@ -1146,7 +1147,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments,
|
||||
tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask],
|
||||
expectedMatrixSize: 30,
|
||||
definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28" },
|
||||
definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", piFileEvidence: "seed-edit-execute-public-download-v1" },
|
||||
},
|
||||
{
|
||||
id: "instruction-persistence", label: "Instruction Persistence",
|
||||
@@ -1489,7 +1490,8 @@ export function buildRunnerMatrix(
|
||||
suiteDefinitionHash: suiteDefinitionHash(suite),
|
||||
profile,
|
||||
environment,
|
||||
task,
|
||||
task: suite.id === "extended-harnesses" && profile.qualificationCandidate === "pi" && task.id === "file-edit-validate"
|
||||
? { ...task, buildPrompt: piFilePrompt } : task,
|
||||
groups: [
|
||||
...new Set([
|
||||
...suite.groups,
|
||||
|
||||
@@ -124,18 +124,24 @@ describe("Pi controls catalog admission", () => {
|
||||
expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true);
|
||||
expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]);
|
||||
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false);
|
||||
expect(validateRunnerCatalog()).toHaveLength(466);
|
||||
expect(validateRunnerCatalog()).toHaveLength(468);
|
||||
for (const cell of cells) expect(JSON.parse(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "pi", model: cell.profile.model }]);
|
||||
expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow();
|
||||
});
|
||||
it("keeps existing Pi-native and active Stop definition identities", () => {
|
||||
it("pins the Pi 1 profile and versioned coverage while retaining active Stop identity", () => {
|
||||
// Pi 1/profile 11 changes profile-bearing definitions. Coverage v3/v2 adds
|
||||
// pending restart and the strict file oracle; no runtime admission is promoted.
|
||||
const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile;
|
||||
expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter");
|
||||
expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 3, profileVersion: 11 });
|
||||
expect(runnerSuites.find(s => s.id === "extended-harnesses")!.definitionMetadata).toMatchObject({ version: 2 });
|
||||
const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)]));
|
||||
expect(hashes).toEqual({
|
||||
"pi-native": "7aaacf83924fdfc7163594e08b9a64ac516f9eb8a0c2c96636b6d2fe4f23776a",
|
||||
"pi-native": "00d4edfd0d61772977434389cb601bbdc08173db4a33fd1e153d2e18f488f597",
|
||||
"native-active-stop": "99682b2b106d816a011834fae5a944ed7729958893709d5b83a19b6f595e7e4d",
|
||||
"rich-acp-warm-continuity": "aead43b3f4f8ed285e2d1c30a517a1850633ae42e593f53122bea9e72af92585",
|
||||
"extended-harnesses": "e75cb2222d2f52ec3a2a425211d195d36b95882de2cb9ed0ad93231b7c42768a",
|
||||
"rich-acp-warm-continuity": "25e69f031696aeb459e6722a056ae0802379f9137799648fdd1945881f32f41a",
|
||||
"extended-harnesses": "e50f79cf604bffb2cdee3ad4e8bc1d106045a4f44ef07e877032ff5b8960c8ab",
|
||||
});
|
||||
expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi" && c.suite.id !== "pi-controls")).toHaveLength(19);
|
||||
expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi" && c.suite.id !== "pi-controls")).toHaveLength(21);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,182 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { collectPiFileEvidence, gradePiCopyback, gradePiFileEvidence, piFileContract, seedPiFile } from "./pi-file-evidence.js";
|
||||
import { canonicalProviderEventsFromAcpxRuntimeEvent } from "../../packages/paperclip-runner/src/provider-events.js";
|
||||
import { safeAcpxLocations } from "../../packages/paperclip-runner/src/drivers/acpx/safe-locations.js";
|
||||
import { classifyFailure } from "./failure-classifier.js";
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
|
||||
function fixture() {
|
||||
const c = piFileContract("fixture"), attachmentId = "12345678-1234-1234-1234-123456789abc";
|
||||
const tool = (seq: number, executionId: string, name: string, operation: string, completed: boolean, target: string | null, output = "") => {
|
||||
// Pi emits absolute edit locations and uses the bash command as title.
|
||||
// Exercise sidecar path normalization followed by the common projection;
|
||||
// direct raw ACP projection would incorrectly lose the workspace target.
|
||||
const canonical = canonicalProviderEventsFromAcpxRuntimeEvent({ type: "tool_call", toolCallId: executionId,
|
||||
tag: completed ? "tool_call_update" : "tool_call", title: name, kind: operation,
|
||||
status: completed ? "completed" : "pending", locations: safeAcpxLocations(target ? [{ path: `/workspace/${target}` }] : [], "/workspace", operation, name), rawOutput: output }, executionId, "turn")[0]!;
|
||||
return { companyId: "company", runId: "run", seq, protocolSchemaVersion: 1, eventType: canonical.eventType,
|
||||
payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "runner",
|
||||
sourceSeq: seq, sourceEventId: `runner:run:${seq}`, runId: "run", turnId: "turn", normalizedSessionId: "session", ...canonical } } };
|
||||
};
|
||||
return {
|
||||
environment: "local" as const, environmentId: "environment", nonce: "fixture", companyId: "company", issueId: "issue", agentId: "agent", attachmentId,
|
||||
seed: { schema: "paperclip.e2e.pi-file-seed.v1", filename: c.filename,
|
||||
beforeSha256: createHash("sha256").update("ready-fixture\n").digest("hex"), byteSize: Buffer.byteLength(c.before) },
|
||||
workspaceBytes: Buffer.from(c.after), downloadedBytes: Buffer.from(c.after),
|
||||
run: { id: "run", companyId: "company", agentId: "agent", runtimeMode: "native", nativeSessionId: "session", runnerInstanceId: "runner", status: "succeeded", resultJson: { semanticToolReceipts: {
|
||||
publish: { operationId: "register_deliverable", input: { filename: c.filename, contentRef: c.filename, contentType: "text/plain", byteSize: c.byteSize, sha256: c.sha256 },
|
||||
result: { disposition: "applied", commandId: `deliverable-prepared:${attachmentId}`, entityRefs: [attachmentId, "product", "comment"] } },
|
||||
} } },
|
||||
events: [tool(1, "edit", "edit", "edit", false, c.filename), tool(2, "edit", "edit", "edit", true, c.filename),
|
||||
tool(3, "validate", c.validationCommand, "execute", false, null), tool(4, "validate", c.validationCommand, "execute", true, null, c.validationMarker)],
|
||||
attachments: [{ id: attachmentId, companyId: "company", issueId: "issue", originatingRunId: "run", createdByAgentId: "agent",
|
||||
originalFilename: c.filename, contentType: "text/plain", byteSize: c.byteSize, sha256: c.sha256 }],
|
||||
activity: [{ action: "issue.attachment_added", companyId: "company", runId: "run", actorId: "agent", entityId: "issue",
|
||||
details: { attachmentId, source: "paperclip_runner_protocol" } }],
|
||||
};
|
||||
}
|
||||
describe("Pi edit, validation and public artifact oracle", () => {
|
||||
it("accepts actual correlated lifecycles, independent bytes and a run-bound registered download", () => {
|
||||
const result = gradePiFileEvidence(fixture());
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.diff.text).toContain("-ready-fixture\n+verified-fixture\n");
|
||||
expect(result.limits.join(" ")).toContain("not attested");
|
||||
expect(result.verification.providerExecutionStatus).toBe("completed");
|
||||
expect(result.verification.commandTitle).toBe(piFileContract("fixture").validationCommand);
|
||||
expect(result.verification.nativeFileAttribution).toBe("workspace_relative_display_target");
|
||||
});
|
||||
const mutations: Array<[string, (f: ReturnType<typeof fixture>) => void]> = [
|
||||
["absent seed", f => { f.seed = {} as any; }],
|
||||
["wrong final bytes", f => { f.workspaceBytes = Buffer.from("wrong\n"); }],
|
||||
["final-only write", f => { f.events[0]!.payload.prpEvent.payload.name = "write"; f.events[1]!.payload.prpEvent.payload.name = "write"; }],
|
||||
["missing edit", f => { f.events = f.events.slice(2); }],
|
||||
["missing validation", f => { f.events = f.events.slice(0, 2); }],
|
||||
["echo-only validation", f => { f.events[2]!.payload.prpEvent.payload.name = `echo ${piFileContract("fixture").validationMarker}`; f.events[3]!.payload.prpEvent.payload.name = f.events[2]!.payload.prpEvent.payload.name; }],
|
||||
["failed validation", f => { f.events[3]!.payload.prpEvent.payload.status = "failed"; }],
|
||||
["unfinished validation", f => { f.events.pop(); }],
|
||||
["unrelated validation turn", f => { f.events[2]!.payload.prpEvent.turnId = "other"; f.events[3]!.payload.prpEvent.turnId = "other"; }],
|
||||
["foreign native session", f => { f.run.nativeSessionId = "other"; }],
|
||||
["foreign native producer", f => { f.run.runnerInstanceId = "other"; }],
|
||||
["foreign event run", f => { f.events[1]!.payload.prpEvent.runId = "other"; }],
|
||||
["duplicate event", f => { f.events.push(structuredClone(f.events[3]!)); }],
|
||||
["missing tool start", f => { f.events.splice(2, 1); }],
|
||||
["truncated validation output", f => { f.events[3]!.payload.prpEvent.payload.outputTruncated = true; }],
|
||||
["missing native target", f => { f.events[0]!.payload.prpEvent.payload.target = null; f.events[1]!.payload.prpEvent.payload.target = null; }],
|
||||
["wrong target", f => { f.events[1]!.payload.prpEvent.payload.target = "other.txt"; }],
|
||||
["missing registration", f => { f.run.resultJson.semanticToolReceipts = {} as any; }],
|
||||
["rejected registration", f => { f.run.resultJson.semanticToolReceipts.publish.result.disposition = "denied"; }],
|
||||
["wrong registered hash", f => { f.run.resultJson.semanticToolReceipts.publish.input.sha256 = "wrong"; }],
|
||||
["wrong registered entity", f => { f.run.resultJson.semanticToolReceipts.publish.result.entityRefs[0] = "other"; }],
|
||||
["missing attachment", f => { f.attachments = []; }],
|
||||
["foreign attachment company", f => { f.attachments[0]!.companyId = "other"; }],
|
||||
["stale attachment run", f => { f.attachments[0]!.originatingRunId = "older"; }],
|
||||
["wrong download bytes", f => { f.downloadedBytes = Buffer.from("wrong\n"); }],
|
||||
["missing publication activity", f => { f.activity = []; }],
|
||||
["foreign publication run", f => { f.activity[0]!.runId = "other"; }],
|
||||
];
|
||||
it.each(mutations)("rejects %s even when the model claims completion", (_name, change) => {
|
||||
const f = fixture(); change(f); expect(() => gradePiFileEvidence(f)).toThrow("Pi file evidence:");
|
||||
try { gradePiFileEvidence(f); } catch (error) { expect(classifyFailure(error)).toBe("candidate_failure"); }
|
||||
});
|
||||
it("keeps real nonce commands within the Rust 240-character name contract", () => {
|
||||
const c = piFileContract("0123456789ab-1");
|
||||
expect([...c.validationCommand].length).toBeLessThanOrEqual(240);
|
||||
expect(safeAcpxLocations([{ path: `/workspace/${c.filename}` }], "/workspace", "edit", "edit")[0]).toMatchObject({
|
||||
path: c.filename, pathBoundary: "paperclip.workspace_relative_display.v2",
|
||||
});
|
||||
expect(() => piFileContract("a".repeat(100))).toThrow("native Rust tool-name bound");
|
||||
});
|
||||
it("runs the exact fixture validation command and rejects plausible wrong bytes", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-validation-command-"));
|
||||
const c = piFileContract("0123456789ab-1");
|
||||
const validate = () => promisify(execFile)("/bin/sh", ["-c", c.validationCommand], {
|
||||
cwd: root, timeout: 3_000, maxBuffer: 4096, env: { PATH: dirname(process.execPath) },
|
||||
});
|
||||
try {
|
||||
await writeFile(join(root, c.filename), c.after);
|
||||
expect((await validate()).stdout).toBe(`${c.validationMarker}\n`);
|
||||
await writeFile(join(root, c.filename), c.after.trimEnd());
|
||||
await expect(validate()).rejects.toMatchObject({ code: 1 });
|
||||
await rm(join(root, c.filename));
|
||||
await expect(validate()).rejects.toMatchObject({ code: 1 });
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
it("changes only Pi file prompts and keeps all four question methods", () => {
|
||||
const cells = runnerMatrix.filter(c => c.suite.id === "extended-harnesses" && c.task.id === "file-edit-validate");
|
||||
expect(cells).toHaveLength(6);
|
||||
for (const cell of cells) {
|
||||
const prompt = cell.task.buildPrompt("fixture");
|
||||
expect(prompt.includes("register_deliverable")).toBe(cell.profile.qualificationCandidate === "pi");
|
||||
}
|
||||
expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi")).toHaveLength(25);
|
||||
});
|
||||
it.each(["local", "daytona"] as const)("seeds once and collects public %s evidence", async environment => {
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-file-evidence-"));
|
||||
try {
|
||||
const seed = await seedPiFile(root, "fixture"), f = environment === "daytona" ? copyback() : fixture(), c = piFileContract("fixture");
|
||||
expect(await readFile(join(root, c.filename), "utf8")).toBe(c.before);
|
||||
await expect(seedPiFile(root, "fixture")).rejects.toThrow();
|
||||
await writeFile(join(root, c.filename), c.after);
|
||||
const get = vi.fn(async (path: string) => {
|
||||
if (path === "/api/heartbeat-runs/run") return f.run;
|
||||
if (path === "/api/environment-leases/lease" && "lease" in f) return f.lease;
|
||||
if (path.endsWith("/attachments")) return f.attachments;
|
||||
if (path.endsWith("/activity")) return f.activity;
|
||||
throw new Error("Unexpected route");
|
||||
});
|
||||
const download = vi.fn(async () => ({ ok: () => true, body: async () => f.downloadedBytes }));
|
||||
const visible = vi.fn(async () => {}), evidence = vi.fn(async () => {});
|
||||
const proof = await collectPiFileEvidence({ ...f, seed, workspace: root, evidence,
|
||||
api: { get, request: { get: download } } as any,
|
||||
page: { locator: () => ({ first: () => ({ waitFor: visible }) }) } as any });
|
||||
expect(proof.passed).toBe(true);
|
||||
if (environment === "daytona") {
|
||||
expect(get).toHaveBeenCalledWith("/api/heartbeat-runs/run");
|
||||
expect(get).toHaveBeenCalledWith("/api/environment-leases/lease");
|
||||
expect(proof.workspaceProvenance.kind).toBe("product_copyback");
|
||||
}
|
||||
expect(visible).toHaveBeenCalledOnce();
|
||||
expect(evidence).toHaveBeenCalledWith(expect.objectContaining({ workspaceBytes: f.workspaceBytes.toString("base64"), downloadedBytes: f.downloadedBytes.toString("base64") }));
|
||||
expect(download).toHaveBeenCalledWith(`/api/attachments/${f.attachmentId}/content`);
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
|
||||
function copyback() {
|
||||
const f = fixture();
|
||||
const reference = { schema: "paperclip.native-workspace-sync/v2", state: "finalized", baselineSha256: "a".repeat(64),
|
||||
descriptorSha256: "b".repeat(64), finalHostSha256: "c".repeat(64), workspaceId: "workspace", leaseId: "lease",
|
||||
providerLeaseId: "sandbox", remoteCwd: "/workspace" };
|
||||
const run = { ...f.run, contextSnapshot: { executionWorkspaceId: "workspace", paperclipEnvironment: { id: "environment", driver: "sandbox", leaseId: "lease" } },
|
||||
runnerProfileJson: { nativeWorkspaceSync: reference, nativeExecutionInput: { binding: { executionWorkspaceId: "workspace" } } } };
|
||||
const lease = { id: "lease", companyId: "company", environmentId: "environment", heartbeatRunId: "run", provider: "daytona", providerLeaseId: "sandbox", metadata: { remoteCwd: "/workspace" } };
|
||||
return { ...f, run, lease, environment: "daytona" as const };
|
||||
}
|
||||
it("accepts public finalized same-run copy-back attribution without claiming guest observation", () => {
|
||||
const f = copyback();
|
||||
expect(gradePiFileEvidence(f).workspaceProvenance).toMatchObject({ kind: "product_copyback", guestObservation: false, hostSnapshotRecomputed: false });
|
||||
});
|
||||
it.each([
|
||||
["missing reference", f => { f.run.runnerProfileJson.nativeWorkspaceSync = {} as any; }],
|
||||
["prepared only", f => { f.run.runnerProfileJson.nativeWorkspaceSync.state = "prepared"; }],
|
||||
["missing baseline", f => { f.run.runnerProfileJson.nativeWorkspaceSync.baselineSha256 = ""; }],
|
||||
["invalid final hash", f => { f.run.runnerProfileJson.nativeWorkspaceSync.finalHostSha256 = "wrong"; }],
|
||||
["foreign workspace", f => { f.run.contextSnapshot.executionWorkspaceId = "other"; }],
|
||||
["contradictory native binding", f => { f.run.runnerProfileJson.nativeExecutionInput.binding.executionWorkspaceId = "other"; }],
|
||||
["foreign lease", f => { f.lease.id = "other"; }],
|
||||
["foreign run", f => { f.lease.heartbeatRunId = "other"; }],
|
||||
["foreign company", f => { f.lease.companyId = "other"; }],
|
||||
["foreign environment", f => { f.lease.environmentId = "other"; }],
|
||||
["foreign provider lease", f => { f.lease.providerLeaseId = "other"; }],
|
||||
["wrong remote root", f => { f.lease.metadata.remoteCwd = "/other"; }],
|
||||
["wrong provider", f => { f.lease.provider = "other"; }],
|
||||
] satisfies Array<[string, (f: ReturnType<typeof copyback>) => void]>)("rejects %s copy-back provenance", (_label, change) => {
|
||||
const f = copyback(); change(f);
|
||||
expect(() => gradePiCopyback(f.run, f.lease, f.companyId, f.environmentId)).toThrow("Pi file evidence:");
|
||||
});
|
||||
@@ -0,0 +1,192 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { Page } from "@playwright/test";
|
||||
import type { RunnerApi } from "./api.js";
|
||||
import { isValidNativePrpEnvelope } from "./native-event-envelope.js";
|
||||
|
||||
type Row = Record<string, any>;
|
||||
const rec = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
|
||||
const sha = (value: string | Buffer) => createHash("sha256").update(value).digest("hex");
|
||||
const requireEvidence = (value: unknown, reason: string) => { if (!value) throw new Error(`Pi file evidence: ${reason}`); };
|
||||
export function piFileContract(nonce: string) {
|
||||
if (!/^[A-Za-z0-9-]{1,100}$/u.test(nonce)) throw new Error("Invalid file fixture nonce");
|
||||
const filename = `extended-${nonce}.txt`, before = `ready-${nonce}\n`, after = `verified-${nonce}\n`;
|
||||
const validationMarker = `PI-VALIDATED-${nonce}`;
|
||||
const validationCommand = `node -e 'if(require("node:fs").readFileSync(${JSON.stringify(filename)},"utf8")!==${JSON.stringify(after)})throw Error("mismatch");console.log(${JSON.stringify(validationMarker)})'`;
|
||||
if ([...validationCommand].length > 240) throw new Error("Validation command exceeds native Rust tool-name bound");
|
||||
return { filename, before, after, sha256: sha(after), byteSize: Buffer.byteLength(after), validationMarker, validationCommand };
|
||||
}
|
||||
export function piFilePrompt(nonce: string) {
|
||||
const c = piFileContract(nonce);
|
||||
return [
|
||||
`The fixture has already created ${c.filename} in your execution workspace with exactly ${JSON.stringify(c.before)}. Read it, then use your native edit tool once to replace those bytes with exactly ${JSON.stringify(c.after)}. Do not substitute write, bash, or an API for the edit.`,
|
||||
`Run this exact native bash command, without an added shell prefix or suffix:\n${c.validationCommand}\nIt independently reads the file, fails on incorrect bytes, and prints the validation marker only after success.`,
|
||||
`Deliver the edited text file as a downloadable attachment using the assigned register_deliverable semantic tool, with filename and workspace-relative contentRef ${c.filename}, contentType text/plain, byteSize ${c.byteSize}, sha256 ${c.sha256}, title Verified text, and a fresh idempotencyKey. Wait for the real registration receipt. Do not upload a different file or claim publication from a disk path.`,
|
||||
`After successful validation and registration, call paperclip_finish with reportedWorkDisposition done, summary EXTENDED-FILE-${nonce}, the current completion contract revision and satisfied objective criterion, no remaining work, the actual validation command/result, and the registered artifact reference.`,
|
||||
`Wait for paperclip_finish to succeed, then emit exactly EXTENDED-FILE-${nonce} as your final response. Do not create unrelated files or work.`,
|
||||
].join("\n");
|
||||
}
|
||||
export async function seedPiFile(workspace: string, nonce: string) {
|
||||
const c = piFileContract(nonce);
|
||||
await writeFile(join(workspace, c.filename), c.before, { flag: "wx" });
|
||||
const bytes = await readFile(join(workspace, c.filename));
|
||||
requireEvidence(bytes.equals(Buffer.from(c.before)), "seed bytes changed before provider startup");
|
||||
return { schema: "paperclip.e2e.pi-file-seed.v1", filename: c.filename, beforeSha256: sha(bytes), byteSize: bytes.length };
|
||||
}
|
||||
|
||||
export function gradePiCopyback(run: Row, lease: Row, companyId: string, environmentId: string) {
|
||||
const context = rec(run.contextSnapshot), environment = rec(context.paperclipEnvironment);
|
||||
const reference = rec(rec(run.runnerProfileJson).nativeWorkspaceSync);
|
||||
const binding = rec(rec(rec(run.runnerProfileJson).nativeExecutionInput).binding);
|
||||
requireEvidence(run.companyId === companyId && reference.schema === "paperclip.native-workspace-sync/v2"
|
||||
&& reference.state === "finalized" && [reference.baselineSha256, reference.descriptorSha256, reference.finalHostSha256]
|
||||
.every(value => typeof value === "string" && /^[a-f0-9]{64}$/u.test(value)), "finalized production copy-back hashes required");
|
||||
requireEvidence(typeof reference.workspaceId === "string" && reference.workspaceId.length > 0
|
||||
&& reference.workspaceId === context.executionWorkspaceId
|
||||
&& (binding.executionWorkspaceId === undefined || binding.executionWorkspaceId === reference.workspaceId)
|
||||
&& environment.id === environmentId && environment.driver === "sandbox" && environment.leaseId === reference.leaseId,
|
||||
"copy-back workspace and environment must belong to the actual run");
|
||||
requireEvidence(lease.id === reference.leaseId && lease.companyId === companyId && lease.environmentId === environmentId
|
||||
&& lease.heartbeatRunId === run.id && lease.provider === "daytona" && typeof reference.providerLeaseId === "string"
|
||||
&& reference.providerLeaseId.length > 0 && lease.providerLeaseId === reference.providerLeaseId
|
||||
&& typeof reference.remoteCwd === "string" && reference.remoteCwd.startsWith("/")
|
||||
&& reference.remoteCwd === rec(lease.metadata).remoteCwd, "copy-back must bind the exact run and provider lease");
|
||||
return { kind: "product_copyback", reference, lease: { id: lease.id, companyId, environmentId, heartbeatRunId: run.id,
|
||||
provider: lease.provider, providerLeaseId: lease.providerLeaseId, remoteCwd: reference.remoteCwd },
|
||||
guestObservation: false, hostSnapshotRecomputed: false };
|
||||
}
|
||||
|
||||
/** Only the common public projection is authority here. Native raw arguments,
|
||||
* full diff blocks and typed command exit codes are deliberately not invented. */
|
||||
export function gradePiFileEvidence(input: {
|
||||
nonce: string; companyId: string; issueId: string; agentId: string; run: Row;
|
||||
environment: "local" | "daytona"; environmentId: string; lease?: Row;
|
||||
events: unknown[]; seed: unknown; workspaceBytes: Buffer; attachments: unknown[];
|
||||
attachmentId: string; downloadedBytes: Buffer; activity: unknown[];
|
||||
}) {
|
||||
const c = piFileContract(input.nonce), run = input.run, seed = rec(input.seed);
|
||||
const workspaceProvenance = input.environment === "daytona"
|
||||
? gradePiCopyback(run, rec(input.lease), input.companyId, input.environmentId)
|
||||
: { kind: "independent_local_workspace" };
|
||||
requireEvidence(run.id && run.companyId === input.companyId && run.agentId === input.agentId
|
||||
&& run.runtimeMode === "native" && run.status === "succeeded", "actual native run identity");
|
||||
requireEvidence(seed.schema === "paperclip.e2e.pi-file-seed.v1" && seed.filename === c.filename
|
||||
&& seed.beforeSha256 === sha(c.before) && seed.byteSize === Buffer.byteLength(c.before), "independent pre-edit seed");
|
||||
requireEvidence(input.workspaceBytes.equals(Buffer.from(c.after)), "independent final workspace bytes");
|
||||
requireEvidence(input.events.length > 0 && input.events.length <= 20_000, "bounded run events");
|
||||
const rows = input.events.map(rec).filter(row => rec(row.payload).prpEvent !== undefined);
|
||||
const seen = new Set<string>(), seqs = new Set<number>(), sourceSeqs = new Map<string, number>();
|
||||
const canonical = rows.map(row => {
|
||||
const e = rec(rec(row.payload).prpEvent);
|
||||
requireEvidence(row.companyId === input.companyId && row.runId === run.id && e.runId === run.id
|
||||
&& isValidNativePrpEnvelope(e, row.protocolSchemaVersion) && row.eventType === e.eventType
|
||||
&& Number.isSafeInteger(row.seq) && row.seq > 0 && !seqs.has(row.seq)
|
||||
&& typeof e.sourceInstanceId === "string" && e.sourceInstanceId.length > 0
|
||||
&& Number.isSafeInteger(e.sourceSeq) && e.sourceSeq > (sourceSeqs.get(e.sourceInstanceId) ?? 0)
|
||||
&& e.sourceEventId === `${e.sourceInstanceId}:${e.runId}:${e.sourceSeq}` && !seen.has(e.sourceEventId), "foreign, duplicated, or reordered durable evidence");
|
||||
seqs.add(row.seq); seen.add(e.sourceEventId); sourceSeqs.set(e.sourceInstanceId, e.sourceSeq);
|
||||
return { row, e, p: rec(e.payload) };
|
||||
});
|
||||
const tools = canonical.filter(x => x.e.eventType.startsWith("tool.execution.") && x.p.transport === "builtin");
|
||||
const edits = tools.filter(x => x.e.eventType === "tool.execution.completed" && x.p.operation === "edit");
|
||||
requireEvidence(edits.length === 1, "one native edit required; final-only writes cannot qualify");
|
||||
const edit = edits[0]!;
|
||||
requireEvidence(edit.p.name === "edit" && edit.p.target === c.filename, "exact native edit and relative target required");
|
||||
function lifecycle(end: typeof edit) {
|
||||
const matches = tools.filter(x => x.p.executionId === end.p.executionId);
|
||||
const starts = matches.filter(x => x.e.eventType === "tool.execution.started");
|
||||
requireEvidence(typeof end.p.executionId === "string" && end.p.executionId.length > 0
|
||||
&& end.p.schema === "paperclip.tool.execution.v1" && end.p.status === "completed"
|
||||
&& end.e.sourceKind === "runner" && end.e.sourceInstanceId === run.runnerInstanceId
|
||||
&& end.e.normalizedSessionId === run.nativeSessionId && typeof end.e.turnId === "string" && end.e.turnId.length > 0
|
||||
&& typeof end.e.normalizedSessionId === "string" && end.e.normalizedSessionId.length > 0
|
||||
&& starts.length === 1 && starts[0]!.row.seq < end.row.seq
|
||||
&& matches.filter(x => x.e.eventType === "tool.execution.completed").length === 1
|
||||
&& matches.every(x => x.p.schema === end.p.schema && x.p.name === end.p.name && x.p.operation === end.p.operation
|
||||
&& x.p.target === end.p.target && x.e.sourceKind === "runner" && x.e.turnId === end.e.turnId
|
||||
&& x.e.normalizedSessionId === end.e.normalizedSessionId && x.e.sourceInstanceId === end.e.sourceInstanceId
|
||||
&& (x === end || x.p.status === "running")), "incomplete or unsuccessful native tool lifecycle");
|
||||
return starts[0]!;
|
||||
}
|
||||
lifecycle(edit);
|
||||
const validations = tools.filter(x => x.e.eventType === "tool.execution.completed" && x.p.operation === "execute"
|
||||
&& x.p.name === c.validationCommand && typeof x.p.output === "string" && x.p.output.includes(c.validationMarker));
|
||||
requireEvidence(validations.length === 1, "one observed provider validation execution");
|
||||
const validation = validations[0]!, start = lifecycle(validation);
|
||||
requireEvidence(start.row.seq > edit.row.seq && validation.e.turnId === edit.e.turnId
|
||||
&& validation.e.normalizedSessionId === edit.e.normalizedSessionId
|
||||
&& validation.e.sourceInstanceId === edit.e.sourceInstanceId && validation.p.outputTruncated === false,
|
||||
"validation must follow the edit in the same native turn");
|
||||
const attachments = input.attachments.map(rec).filter(a => a.id === input.attachmentId);
|
||||
requireEvidence(attachments.length === 1, "one public attachment");
|
||||
const a = attachments[0]!;
|
||||
requireEvidence(a.companyId === input.companyId && a.issueId === input.issueId && a.originatingRunId === run.id
|
||||
&& a.createdByAgentId === input.agentId && a.originalFilename === c.filename && a.contentType === "text/plain"
|
||||
&& a.byteSize === c.byteSize && a.sha256 === c.sha256, "attachment bytes and immutable run attribution");
|
||||
requireEvidence(input.downloadedBytes.equals(Buffer.from(c.after)), "actual public download bytes");
|
||||
const receipts = Object.values(rec(rec(run.resultJson).semanticToolReceipts)).map(rec)
|
||||
.filter(r => r.operationId === "register_deliverable" && rec(r.input).filename === c.filename);
|
||||
requireEvidence(receipts.length === 1, "one authenticated semantic registration");
|
||||
const receipt = receipts[0]!, args = rec(receipt.input), result = rec(receipt.result);
|
||||
requireEvidence(args.contentRef === c.filename && args.contentType === "text/plain" && args.byteSize === c.byteSize
|
||||
&& args.sha256 === c.sha256 && result.disposition === "applied" && result.commandId === `deliverable-prepared:${a.id}`
|
||||
&& Array.isArray(result.entityRefs) && result.entityRefs[0] === a.id, "registered artifact receipt must match downloaded attachment");
|
||||
const publications = input.activity.map(rec).filter(row => row.action === "issue.attachment_added"
|
||||
&& rec(row.details).attachmentId === a.id);
|
||||
requireEvidence(publications.length === 1 && publications[0]!.companyId === input.companyId
|
||||
&& publications[0]!.runId === run.id && publications[0]!.actorId === input.agentId
|
||||
&& publications[0]!.entityId === input.issueId && rec(publications[0]!.details).source === "paperclip_runner_protocol",
|
||||
"publication activity must belong to this run and task");
|
||||
return {
|
||||
schema: "paperclip.e2e.pi-file-evidence.v1", passed: true, workspaceProvenance,
|
||||
runId: run.id, turnId: edit.e.turnId, nativeEditExecutionId: edit.p.executionId,
|
||||
validationExecutionId: validation.p.executionId, attachmentId: a.id,
|
||||
verification: { kind: "independent_exact_bytes", beforeSha256: sha(c.before), afterSha256: c.sha256,
|
||||
downloadedSha256: sha(input.downloadedBytes), byteSize: c.byteSize, providerExecutionStatus: validation.p.status,
|
||||
commandTitle: validation.p.name, nativeFileTarget: edit.p.target,
|
||||
nativeFileAttribution: "workspace_relative_display_target" },
|
||||
diff: { source: "independent_seed_and_workspace_bytes", path: c.filename,
|
||||
text: `--- ${c.filename}\n+++ ${c.filename}\n@@ -1 +1 @@\n-${c.before.trimEnd()}\n+${c.after.trimEnd()}\n` },
|
||||
limits: ["Validation command title is checked exactly; raw arguments and typed exitCode are not projected, so raw invocation/exit code are not attested.",
|
||||
"Diff is computed from independently checked workspace bytes; it does not qualify native diff presentation or private invocation metadata."],
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectPiFileEvidence(input: {
|
||||
page: Page; api: RunnerApi; nonce: string; companyId: string; issueId: string; agentId: string;
|
||||
run: Row; events: unknown[]; seed: unknown; workspace: string;
|
||||
environment: "local" | "daytona"; environmentId: string;
|
||||
evidence(data: unknown): Promise<void>;
|
||||
}) {
|
||||
const c = piFileContract(input.nonce);
|
||||
let run = input.run, lease: Row | undefined;
|
||||
if (input.environment === "daytona") {
|
||||
run = await input.api.get<Row>(`/api/heartbeat-runs/${input.run.id}`);
|
||||
requireEvidence(run.id === input.run.id, "public copy-back run readback changed identity");
|
||||
const leaseId = rec(rec(run.runnerProfileJson).nativeWorkspaceSync).leaseId;
|
||||
requireEvidence(typeof leaseId === "string" && /^[A-Za-z0-9_-]{1,128}$/u.test(leaseId), "copy-back lease reference missing");
|
||||
lease = await input.api.get<Row>(`/api/environment-leases/${leaseId}`);
|
||||
}
|
||||
const attachments = await input.api.get<Row[]>(`/api/issues/${input.issueId}/attachments`);
|
||||
const selected = attachments.filter(a => a.originalFilename === c.filename);
|
||||
requireEvidence(selected.length === 1 && /^[a-f0-9-]{36}$/u.test(selected[0]!.id), "exact published file selection");
|
||||
requireEvidence(selected[0]!.byteSize === c.byteSize && selected[0]!.sha256 === c.sha256
|
||||
&& selected[0]!.contentType === "text/plain", "bounded attachment metadata before download");
|
||||
const attachmentId = selected[0]!.id;
|
||||
const contentPath = `/api/attachments/${attachmentId}/content`;
|
||||
// Exercise the real task surface and the authenticated public download route.
|
||||
await input.page.locator(`a[href*="${contentPath}"]`).first().waitFor({ state: "visible", timeout: 15_000 });
|
||||
const response = await input.api.request.get(contentPath);
|
||||
requireEvidence(response.ok(), "public attachment download failed");
|
||||
const downloadedBytes = await response.body();
|
||||
requireEvidence(downloadedBytes.length <= 4096, "bounded text attachment");
|
||||
const workspaceBytes = await readFile(join(input.workspace, c.filename));
|
||||
const activity = await input.api.get<Row[]>(`/api/issues/${input.issueId}/activity`);
|
||||
await input.evidence({ schema: "paperclip.e2e.pi-file-observation.v1", attachments, attachmentId, activity,
|
||||
workspaceBytes: workspaceBytes.toString("base64"), downloadedBytes: downloadedBytes.toString("base64"),
|
||||
encoding: "base64", expectedSha256: c.sha256, copybackReference: rec(run.runnerProfileJson).nativeWorkspaceSync,
|
||||
copybackLease: lease && { id: lease.id, companyId: lease.companyId, environmentId: lease.environmentId,
|
||||
heartbeatRunId: lease.heartbeatRunId, provider: lease.provider, providerLeaseId: lease.providerLeaseId, remoteCwd: rec(lease.metadata).remoteCwd } });
|
||||
return gradePiFileEvidence({ ...input, run, lease, attachments, attachmentId, downloadedBytes, workspaceBytes, activity });
|
||||
}
|
||||
@@ -5,19 +5,21 @@ import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
|
||||
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
|
||||
|
||||
describe("Pi native Product qualification", () => {
|
||||
it("selects four local and three remote Pi cases without changing the basic extended matrix", () => {
|
||||
it("selects five local and four remote Pi cases without changing the basic extended matrix", () => {
|
||||
const suite = runnerSuites.find(row => row.id === "pi-native")!;
|
||||
expect(suite.manualOnly).toBe(true); expect(suite.expectedMatrixSize).toBe(7);
|
||||
expect(suite.manualOnly).toBe(true); expect(suite.expectedMatrixSize).toBe(9);
|
||||
const cells = runnerMatrix.filter(row => row.suite.id === suite.id);
|
||||
expect(cells).toHaveLength(7);
|
||||
expect(cells).toHaveLength(9);
|
||||
expect(cells.every(row => row.profile.qualificationCandidate === "pi")).toBe(true);
|
||||
expect(cells.filter(row => row.environment.id === "local").map(row => [row.task.id, row.task.expectedRunCount])).toEqual([
|
||||
["native-questions", 1], ["agent-files-fresh-run", 2], ["restrictive-denial", 1], ["human-permission-denial", 1],
|
||||
["native-pending-controller-restart", 1],
|
||||
]);
|
||||
expect(cells.filter(row => row.environment.id === "daytona").map(row => [row.task.id, row.task.expectedRunCount])).toEqual([
|
||||
["native-questions", 1], ["agent-files-fresh-run", 2], ["human-permission-denial", 1],
|
||||
["native-pending-controller-restart", 1],
|
||||
]);
|
||||
expect(cells.reduce((sum, row) => sum + row.task.expectedRunCount, 0)).toBe(9);
|
||||
expect(cells.reduce((sum, row) => sum + row.task.expectedRunCount, 0)).toBe(11);
|
||||
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.suite.id === suite.id)).toBe(false);
|
||||
expect(runnerMatrix.filter(row => row.suite.id === "extended-harnesses")).toHaveLength(30);
|
||||
expect(piNativeTasks[0]!.buildPrompt("fixture")).toContain("paperclip_native_question");
|
||||
|
||||
@@ -6,6 +6,7 @@ export const piNativeTasks: readonly RunnerTaskFixture[] = [
|
||||
["agent-files-fresh-run", "Agent files save and survive a fresh task", 2],
|
||||
["restrictive-denial", "Restrictive native write denial prevents a file effect", 1],
|
||||
["human-permission-denial", "Browser denial prevents a native write through process retirement", 1],
|
||||
["native-pending-controller-restart", "Pending native question survives controller restart in the same live run", 1],
|
||||
].map(([id, label, turns]) => ({
|
||||
id: String(id), label: String(label), groups: [], workMode: "standard", flow: "pi_native",
|
||||
expectedRunCount: Number(turns), attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 },
|
||||
@@ -21,6 +22,11 @@ export function piNativeFinish(marker: string): string {
|
||||
}
|
||||
|
||||
export function piNativePrompt(id: string, nonce: string): string {
|
||||
if (id === "native-pending-controller-restart") return [
|
||||
'Call paperclip_native_question exactly once with {method:"input",title:"Pi native restart",placeholder:"Answer"}. Wait for the real answer; never guess it or substitute semantic request_human_input.',
|
||||
"Use native write to save the exact returned tool-result object as JSON to pi-native-restart-answer.json in the task workspace. Do not change or infer its fields. This is an internal verification file, not a deliverable.",
|
||||
piNativeFinish(`PI-NATIVE-${id}-${nonce}`),
|
||||
].join("\n");
|
||||
if (id === "native-questions") return [
|
||||
"Use paperclip_native_question exactly four times, sequentially, waiting for each answer. Do not substitute semantic request_human_input.",
|
||||
'First: {method:"select",title:"Pi native color",options:[{id:"blue",label:"Blue"},{id:"red",label:"Red"}]}.',
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { LiveFixtureValues } from "./live-fixtures.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
|
||||
import type { RemoteNativeFixture, RemoteNativeSnapshot } from "./remote-native-fixtures.js";
|
||||
import { runPiPendingControllerRestart } from "./pi-native-restart-flow.js";
|
||||
|
||||
export interface PiRemoteBootstrap {
|
||||
prompt(nonce: string): string;
|
||||
@@ -92,7 +93,17 @@ export async function runPiNativeFlow(input: {
|
||||
return result;
|
||||
}
|
||||
try {
|
||||
if (execution.task.id === "native-questions") {
|
||||
if (execution.task.id === "native-pending-controller-restart") {
|
||||
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: ["pi-native-restart-answer.json"] });
|
||||
checks.push(...await runPiPendingControllerRestart({
|
||||
page, companyId: fixtures.company.id, deadlineAt: input.deadlineAt, load, events,
|
||||
restart: input.restart, settle: () => settle(1), capture: input.capture, evidence: input.evidence,
|
||||
readProof: async () => {
|
||||
if (remote) await finishRemote("native-restart-final");
|
||||
return JSON.parse(await readWorkspace("pi-native-restart-answer.json"));
|
||||
},
|
||||
}));
|
||||
} else if (execution.task.id === "native-questions") {
|
||||
const name = `name-${randomBytes(8).toString("hex")}`; const draft = `draft-${randomBytes(8).toString("hex")}\nSecond line`;
|
||||
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: ["pi-native-answers.json"] });
|
||||
const seen = new Set<string>(); let runId: string | undefined;
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { gradePiRestartCompletion, observePiRestartPending, runPiPendingControllerRestart } from "./pi-native-restart-flow.js";
|
||||
import { validatePrpStructuredRunResult } from "../../packages/paperclip-runner/src/protocol/replay-contract.js";
|
||||
|
||||
type Row = Record<string, any>;
|
||||
vi.mock("@playwright/test", () => ({ expect: (locator: { count(): number }) => ({
|
||||
toHaveCount: async (expected: number) => expect(locator.count()).toBe(expected),
|
||||
}) }));
|
||||
|
||||
function fixture(adapter = "acpx-runtime-sidecar") {
|
||||
const questionSet = { schema: "paperclip.question_set.v1", questions: [{ id: "answer", header: "Pi native restart", prompt: "Answer", answerMode: "text", required: true }] };
|
||||
const request = { schema: "paperclip.runtime_request.v2", requestKind: "runtime", type: "input", status: "pending", requestId: "request", turnId: "turn", itemId: "request",
|
||||
input: questionSet, origin: { adapter, provider: "pi", method: "elicitation/create" } };
|
||||
const wrap = (eventType: string, sourceSeq: number, payload: Row): Row => ({
|
||||
companyId: "company", runId: "run", seq: sourceSeq, protocolSchemaVersion: 1, eventType,
|
||||
payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", eventType, runId: "run", turnId: "turn", itemId: "request",
|
||||
normalizedSessionId: "session", sourceInstanceId: "runner", sourceSeq, sourceEventId: `runner:run:${sourceSeq}`, payload } },
|
||||
});
|
||||
const state = { issue: { id: "issue", companyId: "company", status: "in_progress" },
|
||||
runs: [{ id: "run", companyId: "company", nativeIssueId: "issue", runtimeMode: "native", status: "running", nativeSessionId: "session", runnerInstanceId: "runner" } as Row],
|
||||
interactions: [{ id: "interaction", companyId: "company", issueId: "issue", kind: "ask_user_questions", status: "pending", result: null,
|
||||
sourceRunId: "run", continuationPolicy: "none", resolverPolicy: "human_only", idempotencyKey: "paperclip-runner-question:run:request",
|
||||
payload: { runtimeRequestId: "request", questionSet } } as Row] };
|
||||
const events = [wrap("runtime_request.created", 1, { request })];
|
||||
const control = (eventType: string, sourceSeq: number, payload: Row): Row => {
|
||||
const row = wrap(eventType, sourceSeq + 4, payload), event = row.payload.prpEvent;
|
||||
Object.assign(event, { sourceKind: "control_plane", sourceInstanceId: "runner:control", sourceSeq, sourceEventId: `runner:control:run:${sourceSeq}` });
|
||||
delete event.itemId;
|
||||
Object.assign(row, { sourceInstanceId: event.sourceInstanceId, sourceSeq, sourceEventId: event.sourceEventId });
|
||||
return row;
|
||||
};
|
||||
function finish(answer = "hidden") {
|
||||
state.issue.status = "done"; state.runs[0]!.status = "succeeded";
|
||||
Object.assign(state.interactions[0]!, { status: "answered", result: { answers: [{ questionId: "answer", optionIds: [], otherText: answer }] } });
|
||||
const validated = validatePrpStructuredRunResult({ summary: "Restart question answered", reportedWorkDisposition: "done",
|
||||
completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, evidence: [], verification: [] });
|
||||
if (!validated.ok) throw new Error("Fixture finish result must satisfy the actual production schema");
|
||||
const result = validated.result;
|
||||
events.push(wrap("runtime_request.resolved", 2, { requestId: "request", turnId: "turn", itemId: "request", action: "submit",
|
||||
response: { schema: "paperclip.question_response.v1", answers: { answer: { text: answer } } } }), wrap("turn.completed", 4, { status: "completed" }),
|
||||
wrap("run.result.proposed", 3, structuredClone(result)), control("run.result.accepted", 1, { result: structuredClone(result) }),
|
||||
control("run.terminal", 2, { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }));
|
||||
return { status: "answered", value: answer };
|
||||
}
|
||||
return { state, events, request, wrap, control, finish };
|
||||
}
|
||||
|
||||
it.each(["acpx-runtime", "acpx-runtime-sidecar"])("accepts mixed runner/control-plane production completion via %s", adapter => {
|
||||
const f = fixture(adapter), pending = observePiRestartPending(f.state, f.events, "company"), proof = f.finish();
|
||||
expect(gradePiRestartCompletion(f.state, f.events, pending, "hidden", proof)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["no native event", (f: ReturnType<typeof fixture>) => { f.events.length = 0; }],
|
||||
["wrong origin", (f: ReturnType<typeof fixture>) => { f.request.origin.provider = "cursor"; }],
|
||||
["semantic origin", (f: ReturnType<typeof fixture>) => { f.request.origin.adapter = "semantic"; }],
|
||||
["internal kind instead of public schema", (f: ReturnType<typeof fixture>) => { f.request.requestKind = "elicitation"; }],
|
||||
["missing native session", (f: ReturnType<typeof fixture>) => { delete f.state.runs[0]!.nativeSessionId; }],
|
||||
["foreign request turn", (f: ReturnType<typeof fixture>) => { f.request.turnId = "other"; }],
|
||||
["foreign company", (f: ReturnType<typeof fixture>) => { f.state.interactions[0]!.companyId = "other"; }],
|
||||
["durable fallback", (f: ReturnType<typeof fixture>) => { f.state.interactions[0]!.continuationPolicy = "wake_assignee"; }],
|
||||
["wrong idempotency identity", (f: ReturnType<typeof fixture>) => { f.state.interactions[0]!.idempotencyKey = "other"; }],
|
||||
["duplicate interaction", (f: ReturnType<typeof fixture>) => { f.state.interactions.push(structuredClone(f.state.interactions[0]!)); }],
|
||||
["duplicate source receipt", (f: ReturnType<typeof fixture>) => { f.events.push(structuredClone(f.events[0]!)); }],
|
||||
["already answered", (f: ReturnType<typeof fixture>) => { f.finish(); }],
|
||||
] as const)("rejects %s before restart", (_label, mutate) => {
|
||||
const f = fixture(); mutate(f); expect(() => observePiRestartPending(f.state, f.events, "company")).toThrow("Pi native restart");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["replacement run", (f: ReturnType<typeof fixture>) => { f.state.runs[0]!.id = "replacement"; }],
|
||||
["additional run", (f: ReturnType<typeof fixture>) => { f.state.runs.push({ ...f.state.runs[0], id: "replayed" }); }],
|
||||
["replacement native session", (f: ReturnType<typeof fixture>) => { f.state.runs[0]!.nativeSessionId = "replacement"; }],
|
||||
["replacement producer", (f: ReturnType<typeof fixture>) => { f.state.runs[0]!.runnerInstanceId = "replacement"; }],
|
||||
["replaced interaction", (f: ReturnType<typeof fixture>) => { f.state.interactions[0]!.id = "replacement"; }],
|
||||
["missing resolution", (f: ReturnType<typeof fixture>) => { f.events.splice(1, 1); }],
|
||||
["duplicate resolution", (f: ReturnType<typeof fixture>) => { f.events.push(structuredClone(f.events[1]!)); }],
|
||||
["replayed creation", (f: ReturnType<typeof fixture>) => { f.events.push(f.wrap("runtime_request.created", 4, { request: f.request })); }],
|
||||
["cancelled callback", (f: ReturnType<typeof fixture>) => { f.events[1]!.eventType = f.events[1]!.payload.prpEvent.eventType = "runtime_request.cancelled"; }],
|
||||
["expired callback", (f: ReturnType<typeof fixture>) => { f.events[1]!.eventType = f.events[1]!.payload.prpEvent.eventType = "runtime_request.expired"; }],
|
||||
["wrong delivered answer", (f: ReturnType<typeof fixture>) => { f.events[1]!.payload.prpEvent.payload.response.answers.answer.text = "guess"; }],
|
||||
["wrong durable answer", (f: ReturnType<typeof fixture>) => { f.state.interactions[0]!.result.answers[0].otherText = "guess"; }],
|
||||
["wrong response turn", (f: ReturnType<typeof fixture>) => { f.events[1]!.payload.prpEvent.payload.turnId = "other"; }],
|
||||
["missing resolution envelope turn", (f: ReturnType<typeof fixture>) => { delete f.events[1]!.payload.prpEvent.turnId; }],
|
||||
["missing resolution item", (f: ReturnType<typeof fixture>) => { delete f.events[1]!.payload.prpEvent.itemId; }],
|
||||
["rewritten canonical question", (f: ReturnType<typeof fixture>) => { f.request.input.questions[0]!.prompt = "Replacement prompt"; }],
|
||||
["new native turn", (f: ReturnType<typeof fixture>) => { f.events[2]!.payload.prpEvent.turnId = "other"; }],
|
||||
["failed terminal payload", (f: ReturnType<typeof fixture>) => { f.events[2]!.payload.prpEvent.payload.status = "failed"; }],
|
||||
["missing terminal", (f: ReturnType<typeof fixture>) => { f.events.splice(2, 1); }],
|
||||
["duplicate terminal", (f: ReturnType<typeof fixture>) => { f.events.push(f.wrap("turn.completed", 4, {})); }],
|
||||
["missing control-plane acceptance", (f: ReturnType<typeof fixture>) => { f.events.splice(4, 1); }],
|
||||
["missing control-plane terminal", (f: ReturnType<typeof fixture>) => { f.events.pop(); }],
|
||||
["foreign control-plane source", (f: ReturnType<typeof fixture>) => { f.events[4]!.payload.prpEvent.sourceInstanceId = "other:control"; }],
|
||||
["foreign control-plane turn", (f: ReturnType<typeof fixture>) => { f.events[4]!.payload.prpEvent.turnId = "other"; }],
|
||||
["missing control-plane turn", (f: ReturnType<typeof fixture>) => { delete f.events[4]!.payload.prpEvent.turnId; }],
|
||||
["foreign control-plane session", (f: ReturnType<typeof fixture>) => { f.events[4]!.payload.prpEvent.normalizedSessionId = "other"; }],
|
||||
["mismatched durable control source", (f: ReturnType<typeof fixture>) => { f.events[4]!.sourceSeq = 5; }],
|
||||
["duplicate control receipt", (f: ReturnType<typeof fixture>) => { f.events.push(structuredClone(f.events[4]!)); }],
|
||||
["control plane impersonates a native turn", (f: ReturnType<typeof fixture>) => { f.events.push(f.control("turn.completed", 3, { status: "completed" })); }],
|
||||
["failed control terminal", (f: ReturnType<typeof fixture>) => { f.events[5]!.payload.prpEvent.payload.runTerminalState = "failed"; }],
|
||||
["unaccepted native result", (f: ReturnType<typeof fixture>) => { f.events[4]!.payload.prpEvent.payload.result.summary = "Replacement result"; }],
|
||||
["malformed accepted result", (f: ReturnType<typeof fixture>) => { delete f.events[4]!.payload.prpEvent.payload.result.schema; }],
|
||||
["runner impersonates acceptance", (f: ReturnType<typeof fixture>) => { f.events[4]!.payload.prpEvent.sourceKind = "runner"; }],
|
||||
] as const)("rejects %s despite a correct workspace answer", (_label, mutate) => {
|
||||
const f = fixture(), pending = observePiRestartPending(f.state, f.events, "company"), proof = f.finish();
|
||||
mutate(f); expect(() => gradePiRestartCompletion(f.state, f.events, pending, "hidden", proof)).toThrow();
|
||||
});
|
||||
|
||||
it.each([null, {}, { status: "answered", value: "guess" }, { status: "cancelled", value: "hidden" }, { status: "answered", value: "hidden", invented: true }])(
|
||||
"rejects missing or wrong independent file proof %j", proof => {
|
||||
const f = fixture(), pending = observePiRestartPending(f.state, f.events, "company"); f.finish();
|
||||
expect(() => gradePiRestartCompletion(f.state, f.events, pending, "hidden", proof)).toThrow();
|
||||
},
|
||||
);
|
||||
|
||||
async function flow(failure?: "replaced" | "wrong-file" | "missing-resolution") {
|
||||
const f = fixture(), evidence = new Map<string, any>(), checkpoints: string[] = [];
|
||||
let answer = "", restarts = 0, submissions = 0, proof: unknown, resolvePost: ((value: unknown) => void) | undefined;
|
||||
let postPredicate: ((value: any) => boolean) | undefined;
|
||||
const composer: any = { count: () => 1, filter: () => composer, locator: () => composer, first: () => composer,
|
||||
fill: async (value: string) => { expect(restarts).toBe(1); answer = value; } };
|
||||
const button: any = { count: () => 1, filter: () => button, click: async () => {
|
||||
expect(restarts).toBe(1); expect(answer).toMatch(/^PI-RESTART-[a-f0-9]{32}$/); submissions++;
|
||||
const request = { url: () => "http://fixture/api/issues/issue/interactions/interaction/respond", method: () => "POST",
|
||||
postDataJSON: () => ({ answers: [{ questionId: "answer", optionIds: [], otherText: answer }] }) };
|
||||
expect(postPredicate!(request)).toBe(true); resolvePost!(request); proof = f.finish(answer);
|
||||
if (failure === "missing-resolution") f.events.splice(1, 1);
|
||||
} };
|
||||
const page: any = { getByTestId: () => composer, getByRole: () => button, reload: async () => { checkpoints.push("reload"); },
|
||||
waitForRequest: (predicate: (value: any) => boolean) => { postPredicate = predicate; return new Promise(resolve => { resolvePost = resolve; }); } };
|
||||
const result = runPiPendingControllerRestart({ page, companyId: "company", deadlineAt: Date.now() + 1000,
|
||||
load: async () => structuredClone(f.state), events: async () => structuredClone(f.events),
|
||||
restart: async () => {
|
||||
expect(submissions).toBe(0); expect(f.state.runs[0]!.status).toBe("running"); expect(f.state.interactions[0]!.status).toBe("pending");
|
||||
expect(answer).toBe(""); restarts++; checkpoints.push("restart");
|
||||
if (failure === "replaced") f.state.interactions[0]!.id = "replacement";
|
||||
},
|
||||
settle: async () => structuredClone(f.state), readProof: async () => failure === "wrong-file" ? { status: "answered", value: "guess" } : proof,
|
||||
capture: async id => { checkpoints.push(id); }, evidence: async (name, data) => { evidence.set(name, structuredClone(data)); },
|
||||
});
|
||||
if (failure) await expect(result).rejects.toThrow("Pi native restart");
|
||||
else expect((await result).every(check => check.passed)).toBe(true);
|
||||
expect(restarts).toBe(1); expect(submissions).toBe(failure === "replaced" ? 0 : 1);
|
||||
expect(checkpoints.slice(0, 3)).toEqual(["pi-restart-pending", "restart", "reload"]);
|
||||
expect(evidence.has("pi-native-restart-before.json")).toBe(true);
|
||||
expect(evidence.has("pi-native-restart-after.json")).toBe(true);
|
||||
expect(evidence.has("pi-native-restart-checks.json")).toBe(true);
|
||||
expect(JSON.stringify(evidence.get("pi-native-restart-before.json"))).not.toContain("PI-RESTART-");
|
||||
if (failure !== "replaced") expect(evidence.has("pi-native-restart-final.json")).toBe(true);
|
||||
}
|
||||
it("restarts while unanswered, reloads, then submits hidden text through the exact browser route", () => flow());
|
||||
it.each(["replaced", "wrong-file", "missing-resolution"] as const)("retains evidence and fails whole flow on %s", failure => flow(failure));
|
||||
@@ -0,0 +1,158 @@
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
import { canonicalJson } from "../../packages/shared/src/portability-hash.js";
|
||||
import { validatePrpStructuredRunResult } from "../../packages/paperclip-runner/src/protocol/replay-contract.js";
|
||||
import { pollUntil } from "./api.js";
|
||||
import { hasAcpxNativeOrigin } from "./acpx-native-origin.js";
|
||||
import { isValidNativePrpEnvelope } from "./native-event-envelope.js";
|
||||
|
||||
type Row = Record<string, any>;
|
||||
type State = { issue: Row; runs: Row[]; interactions: Row[] };
|
||||
type Check = { id: string; passed: boolean; detail: string };
|
||||
const rec = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
|
||||
const id = (value: unknown): value is string => typeof value === "string" && value.length > 0 && value.length <= 240 && !/[\u0000-\u001f\u007f]/u.test(value) && !value.includes("[REDACTED]");
|
||||
const digest = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex");
|
||||
const closures = new Set(["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"]);
|
||||
const terminals = new Set(["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]);
|
||||
function requireProof(value: unknown, reason: string): asserts value { if (!value) throw new Error(`Pi native restart: ${reason}`); }
|
||||
|
||||
export interface PiRestartPending {
|
||||
companyId: string; issueId: string; runId: string; interactionId: string; requestId: string;
|
||||
turnId: string; itemId: string; nativeSessionId: string; sourceInstanceId: string; questionId: string;
|
||||
createdSourceSeq: number; createdRowSha256: string; interactionBindingSha256: string;
|
||||
}
|
||||
|
||||
function interactionBinding(card: Row) {
|
||||
return { id: card.id, companyId: card.companyId, issueId: card.issueId, kind: card.kind,
|
||||
sourceRunId: card.sourceRunId, idempotencyKey: card.idempotencyKey,
|
||||
continuationPolicy: card.continuationPolicy, resolverPolicy: card.resolverPolicy, payload: card.payload };
|
||||
}
|
||||
|
||||
function inspect(state: State, events: readonly Row[], companyId: string) {
|
||||
requireProof(id(companyId) && id(state.issue.id) && state.issue.companyId === companyId && state.runs.length === 1 && state.interactions.length === 1,
|
||||
"one company-scoped task, run and question required");
|
||||
const run = state.runs[0]!, card = state.interactions[0]!;
|
||||
requireProof(id(run.id) && run.companyId === companyId && run.nativeIssueId === state.issue.id && run.runtimeMode === "native"
|
||||
&& id(run.nativeSessionId) && id(run.runnerInstanceId), "native run/session identity missing");
|
||||
requireProof(id(card.id) && card.companyId === companyId && card.issueId === state.issue.id && card.kind === "ask_user_questions"
|
||||
&& card.sourceRunId === run.id && card.continuationPolicy === "none" && card.resolverPolicy === "human_only", "question is not the native callback");
|
||||
requireProof(Array.isArray(events) && events.length > 0 && events.length <= 20_000, "bounded durable events required");
|
||||
const rows = events.filter(row => rec(row.payload).prpEvent !== undefined).map(row => ({ row, event: rec(rec(row.payload).prpEvent) })).sort((a, b) => a.row.seq - b.row.seq);
|
||||
const seenRows = new Set<number>(), seenSources = new Set<string>(), cursors = new Map<string, number>();
|
||||
for (const { row, event } of rows) {
|
||||
requireProof(row.companyId === companyId && row.runId === run.id && isValidNativePrpEnvelope(event, row.protocolSchemaVersion)
|
||||
&& event.runId === run.id && event.eventType === row.eventType && ["runner", "control_plane"].includes(event.sourceKind)
|
||||
&& Number.isSafeInteger(row.seq) && row.seq > 0 && !seenRows.has(row.seq)
|
||||
&& id(event.sourceInstanceId) && Number.isSafeInteger(event.sourceSeq) && event.sourceSeq > (cursors.get(event.sourceInstanceId) ?? 0)
|
||||
&& event.sourceEventId === `${event.sourceInstanceId}:${run.id}:${event.sourceSeq}` && !seenSources.has(event.sourceEventId),
|
||||
"foreign, duplicate or reordered durable event");
|
||||
// The production finalizer owns this separate, stable two-event stream.
|
||||
// It cannot impersonate a native request/turn, nor can a runner accept itself.
|
||||
if (event.sourceKind === "control_plane") {
|
||||
requireProof(event.sourceInstanceId === `${run.runnerInstanceId}:control` && event.normalizedSessionId === run.nativeSessionId
|
||||
&& row.sourceInstanceId === event.sourceInstanceId && row.sourceSeq === event.sourceSeq && row.sourceEventId === event.sourceEventId
|
||||
&& ((event.eventType === "run.result.accepted" && event.sourceSeq === 1) || (event.eventType === "run.terminal" && event.sourceSeq === 2)),
|
||||
"foreign or unexpected control-plane completion receipt");
|
||||
} else requireProof(!["run.result.accepted", "run.terminal"].includes(event.eventType), "runner cannot claim control-plane completion");
|
||||
seenRows.add(row.seq); seenSources.add(event.sourceEventId); cursors.set(event.sourceInstanceId, event.sourceSeq);
|
||||
}
|
||||
const created = rows.filter(x => x.event.eventType === "runtime_request.created");
|
||||
requireProof(created.length === 1, "exactly one native request creation required");
|
||||
const opening = created[0]!, request = rec(opening.event.payload?.request), questionSet = rec(request.input);
|
||||
const question = Array.isArray(questionSet.questions) && questionSet.questions.length === 1 ? rec(questionSet.questions[0]) : {};
|
||||
requireProof(request.schema === "paperclip.runtime_request.v2" && request.requestKind === "runtime" && request.type === "input" && request.status === "pending"
|
||||
&& id(request.requestId) && id(request.itemId) && id(request.turnId) && request.turnId === opening.event.turnId && request.itemId === opening.event.itemId
|
||||
&& hasAcpxNativeOrigin(request.origin, "pi", "elicitation/create") && questionSet.schema === "paperclip.question_set.v1"
|
||||
&& id(question.id) && question.answerMode === "text" && question.header === "Pi native restart"
|
||||
&& card.payload?.runtimeRequestId === request.requestId && digest(card.payload.questionSet) === digest(questionSet)
|
||||
&& card.idempotencyKey === `paperclip-runner-question:${run.id}:${request.requestId}`, "canonical Pi input request identity missing");
|
||||
requireProof(rows.filter(x => x.event.turnId != null || x.event.eventType.startsWith("turn.") || x.event.eventType.startsWith("runtime_request.")).every(({ event }) => event.turnId === request.turnId
|
||||
&& event.normalizedSessionId === run.nativeSessionId
|
||||
&& event.sourceInstanceId === (event.sourceKind === "runner" ? run.runnerInstanceId : `${run.runnerInstanceId}:control`)), "turn, native session or producer was replaced");
|
||||
requireProof(rows.filter(x => x.event.sourceKind === "control_plane").every(x => x.event.turnId === request.turnId), "control-plane result belongs to another turn");
|
||||
const binding: PiRestartPending = { companyId, issueId: state.issue.id, runId: run.id, interactionId: card.id, requestId: request.requestId,
|
||||
turnId: request.turnId, itemId: request.itemId, nativeSessionId: run.nativeSessionId, sourceInstanceId: run.runnerInstanceId, questionId: question.id,
|
||||
createdSourceSeq: opening.event.sourceSeq, createdRowSha256: digest(opening.row), interactionBindingSha256: digest(interactionBinding(card)) };
|
||||
return { run, card, rows, binding };
|
||||
}
|
||||
|
||||
/** Grade public state and PRP receipts; assistant prose is never restart proof. */
|
||||
export function observePiRestartPending(state: State, events: readonly Row[], companyId: string): PiRestartPending {
|
||||
const { run, card, rows, binding } = inspect(state, events, companyId);
|
||||
requireProof(state.issue.status === "in_progress" && run.status === "running" && card.status === "pending" && card.result == null
|
||||
&& run.resultJson?.nativeCancellation == null && run.resultJson?.startupCancellation == null
|
||||
&& !rows.some(x => closures.has(x.event.eventType) || terminals.has(x.event.eventType) || x.event.sourceKind === "control_plane"), "question is no longer unanswered in a live run");
|
||||
return binding;
|
||||
}
|
||||
|
||||
export function gradePiRestartCompletion(state: State, events: readonly Row[], pending: PiRestartPending, answer: string, proof: unknown): boolean {
|
||||
const { run, card, rows, binding } = inspect(state, events, pending.companyId);
|
||||
requireProof(digest(binding) === digest(pending), "original request/run/source identity changed");
|
||||
const resolved = rows.filter(x => closures.has(x.event.eventType)), terminal = rows.filter(x => terminals.has(x.event.eventType));
|
||||
requireProof(resolved.length === 1 && resolved[0]!.event.eventType === "runtime_request.resolved", "exactly one durable native resolution required");
|
||||
const delivered = resolved[0]!.event, response = rec(delivered.payload);
|
||||
requireProof(response.requestId === pending.requestId && response.turnId === pending.turnId && response.itemId === pending.itemId
|
||||
&& response.action === "submit" && delivered.itemId === pending.itemId && delivered.sourceSeq > pending.createdSourceSeq
|
||||
&& digest(response.response) === digest({ schema: "paperclip.question_response.v1", answers: { [pending.questionId]: { text: answer } } }),
|
||||
"hidden browser answer was not delivered to the original native request");
|
||||
requireProof(terminal.length === 1 && terminal[0]!.event.eventType === "turn.completed" && terminal[0]!.event.payload?.status === "completed"
|
||||
&& terminal[0]!.event.payload?.error == null && terminal[0]!.event.sourceSeq > delivered.sourceSeq
|
||||
&& state.issue.status === "done" && run.status === "succeeded" && run.resultJson?.nativeCancellation == null && run.resultJson?.startupCancellation == null,
|
||||
"original native turn did not finish exactly once");
|
||||
const proposed = rows.filter(x => x.event.eventType === "run.result.proposed"), accepted = rows.filter(x => x.event.eventType === "run.result.accepted"), runTerminal = rows.filter(x => x.event.eventType === "run.terminal");
|
||||
requireProof(proposed.length === 1 && accepted.length === 1 && runTerminal.length === 1, "unique proposed and control-plane accepted completion required");
|
||||
const result = rec(accepted[0]!.event.payload?.result), validation = validatePrpStructuredRunResult(result);
|
||||
requireProof(validation.ok && digest(validation.result) === digest(result) && result.reportedWorkDisposition === "done"
|
||||
&& digest(proposed[0]!.event.payload) === digest(result)
|
||||
&& proposed[0]!.row.seq > resolved[0]!.row.seq && proposed[0]!.row.seq < terminal[0]!.row.seq
|
||||
&& accepted[0]!.row.seq > terminal[0]!.row.seq && runTerminal[0]!.row.seq > accepted[0]!.row.seq
|
||||
&& digest(runTerminal[0]!.event.payload) === digest({ schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }),
|
||||
"control-plane completion does not accept the original successful native result");
|
||||
requireProof(card.status === "answered" && Array.isArray(card.result?.answers) && card.result.answers.length === 1
|
||||
&& card.result.answers[0].questionId === pending.questionId && card.result.answers[0].otherText === answer, "durable interaction answer missing or wrong");
|
||||
requireProof(digest(proof) === digest({ status: "answered", value: answer }), "independent workspace proof is missing or wrong");
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function runPiPendingControllerRestart(input: {
|
||||
page: Page; companyId: string; deadlineAt: number; load(): Promise<State>; events(runId: string): Promise<Row[]>;
|
||||
restart(): Promise<void>; settle(): Promise<State>; readProof(): Promise<unknown>;
|
||||
capture(id: string, label: string, file: string): Promise<void>; evidence(name: string, data: unknown): Promise<void>;
|
||||
}): Promise<Check[]> {
|
||||
const checks: Check[] = [], answer = `PI-RESTART-${randomBytes(16).toString("hex")}`;
|
||||
const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); requireProof(passed, detail); };
|
||||
const snapshot = async (name: string) => {
|
||||
const state = await input.load(), events = state.runs.length === 1 ? await input.events(state.runs[0]!.id) : [];
|
||||
await input.evidence(name, { ...state, events }); return { state, events };
|
||||
};
|
||||
try {
|
||||
await pollUntil({ label: "Pi pending native restart question", deadlineAt: input.deadlineAt, load: input.load,
|
||||
accept: state => state.interactions.some(card => card.status === "pending"),
|
||||
reject: state => state.runs.length > 1 || state.runs.some(run => ["failed", "cancelled", "timed_out", "succeeded"].includes(run.status)) ? "Pi ended or created another run before the pending restart boundary" : undefined });
|
||||
const before = await snapshot("pi-native-restart-before.json"), pending = observePiRestartPending(before.state, before.events, input.companyId);
|
||||
check("restart-pending-native-identity", true, "One unanswered native Pi callback belongs to the original running turn and session");
|
||||
const composer = () => input.page.getByTestId("question-text-answer-composer").filter({ visible: true });
|
||||
await expect(composer()).toHaveCount(1);
|
||||
await input.capture("pi-restart-pending", "Pi native question before controller restart", "pi-restart-pending.png");
|
||||
await input.restart();
|
||||
await input.page.reload();
|
||||
const after = await snapshot("pi-native-restart-after.json"), resumed = observePiRestartPending(after.state, after.events, input.companyId);
|
||||
check("restart-same-pending-native-request", digest(resumed) === digest(pending), "Controller restart retained the same unanswered request, run, turn, native session and producer");
|
||||
await expect(composer()).toHaveCount(1);
|
||||
await input.capture("pi-restart-reconnected", "Same Pi question after controller restart", "pi-restart-reconnected.png");
|
||||
await composer().locator('[contenteditable="true"],textarea').first().fill(answer);
|
||||
const route = `/api/issues/${pending.issueId}/interactions/${pending.interactionId}/respond`;
|
||||
const submitted = input.page.waitForRequest(request => new URL(request.url()).pathname === route && request.method() === "POST",
|
||||
{ timeout: Math.max(1, input.deadlineAt - Date.now()) });
|
||||
const button = input.page.getByRole("button", { name: after.state.interactions[0]!.payload.questionSet.submitLabel ?? "Submit answers", exact: true }).filter({ visible: true });
|
||||
await expect(button).toHaveCount(1); await button.click();
|
||||
const posted = (await submitted).postDataJSON();
|
||||
check("restart-exact-browser-answer", Array.isArray(posted?.answers) && posted.answers.length === 1
|
||||
&& posted.answers[0].questionId === pending.questionId && posted.answers[0].otherText === answer, "Actual browser POST answered the retained question with previously undisclosed text");
|
||||
const final = await input.settle(), events = await input.events(pending.runId), proof = await input.readProof();
|
||||
await input.evidence("pi-native-restart-final.json", { ...final, events, pending, posted, proof });
|
||||
check("restart-native-answer-delivered-once", gradePiRestartCompletion(final, events, pending, answer, proof),
|
||||
"One durable answer and one successful original native turn delivered the exact typed result after restart");
|
||||
return checks;
|
||||
} finally { await input.evidence("pi-native-restart-checks.json", checks); }
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { createRemoteNativeBootstrap, createRemoteFixtureClient } from "./remote
|
||||
import { runCleanupWithObservers, verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js";
|
||||
import { runCopilotProtectionFlow } from "./copilot-protection-flow.js";
|
||||
import { runPiNativeFlow } from "./pi-native-flow.js";
|
||||
import { seedPiFile, collectPiFileEvidence } from "./pi-file-evidence.js";
|
||||
import { warmManagedFileEvidence } from "./warm-managed-files.js";
|
||||
import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js";
|
||||
import { runsCompletionUpdateProbe, completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js";
|
||||
@@ -574,6 +575,7 @@ for (const execution of executions) {
|
||||
let runtimeLeases: EnvironmentLeaseRecord[] = [];
|
||||
let matcherResults: MatcherResult[] = [];
|
||||
let downloadedResponseProof: Awaited<ReturnType<typeof readResponseProof>> | undefined;
|
||||
let piFileSeed: Awaited<ReturnType<typeof seedPiFile>> | undefined;
|
||||
const completionQuality: CompletionQualityRecord[] = [];
|
||||
const completionEvidence = async (name: string, data: unknown) => {
|
||||
await writeSanitizedJson(snapshotsDir, name, data, secrets);
|
||||
@@ -845,6 +847,11 @@ for (const execution of executions) {
|
||||
});
|
||||
expect(experimental.enableNativeRunner).toBe(true);
|
||||
|
||||
if (execution.suite.id === "extended-harnesses" && execution.profile.qualificationCandidate === "pi" && execution.task.id === "file-edit-validate") {
|
||||
piFileSeed = await seedPiFile(workspacePath, nonce);
|
||||
await writeSanitizedJson(snapshotsDir, "pi-file-seed.json", piFileSeed, secrets);
|
||||
}
|
||||
|
||||
if (execution.suite.id === "daytona-git-streaming") {
|
||||
await setupGitStreamingWorkspace(workspacePath);
|
||||
}
|
||||
@@ -2136,6 +2143,15 @@ for (const execution of executions) {
|
||||
await writeSanitizedJson(snapshotsDir, "api-response-pagination.json", paging, secrets);
|
||||
if (!paging.passed) invariantFailures.push(`Bounded response paging was not proven: ${paging.failure}`);
|
||||
}
|
||||
if (piFileSeed) {
|
||||
const proof = await collectPiFileEvidence({ page, api, nonce, companyId: fixtures.company.id,
|
||||
issueId: issue.id, agentId: fixtures.agent.id, run: finalRun,
|
||||
events: runEventsByRun.find(entry => entry.runId === finalRun.id)?.events ?? [],
|
||||
seed: piFileSeed, workspace: workspacePath,
|
||||
environment: execution.environment.id === "daytona" ? "daytona" : "local", environmentId: fixtures.environment.id,
|
||||
evidence: data => writeSanitizedJson(snapshotsDir, "pi-file-observation.json", data, secrets) });
|
||||
await writeSanitizedJson(snapshotsDir, "pi-file-evidence.json", proof, secrets);
|
||||
}
|
||||
if (execution.suite.id === "task-titles") {
|
||||
const titleEvidence = gradeTaskTitle({
|
||||
initial: titleCreation?.issue, submitted: titleCreation?.submitted,
|
||||
|
||||
@@ -46,6 +46,10 @@ function stopper(gracefulTimeoutMs = 2_000) {
|
||||
});
|
||||
return { stop, logs, errors };
|
||||
}
|
||||
function expectSuccessfulWrapperReceipt(wrapper: ChildProcess, messages: unknown[], diagnostics = "") {
|
||||
expect([wrapper.exitCode, wrapper.signalCode], diagnostics).toEqual([0, null]);
|
||||
expect(messages, diagnostics).toContainEqual({ exitCode: 0, signalCode: null });
|
||||
}
|
||||
afterEach(async () => {
|
||||
for (const child of children.splice(0)) {
|
||||
if (child.exitCode !== null || child.signalCode !== null) continue;
|
||||
@@ -153,15 +157,19 @@ it.skipIf(process.platform === "win32")("lets launcher cancellation join wrapper
|
||||
// during the async close even though the helper sends only one signal.
|
||||
const messages: unknown[] = [];
|
||||
wrapper.on("message", message => messages.push(message));
|
||||
const exited = once(wrapper, "exit");
|
||||
// Process exit is not IPC completion. Register both barriers before
|
||||
// cancellation so the final worker receipt is drained before asserting it.
|
||||
const closed = once(wrapper, "close");
|
||||
const disconnected = once(wrapper, "disconnect");
|
||||
await owner.observe();
|
||||
const cancellation = stopOwnedProcessTree(wrapper, owner, 2_000, 2_000);
|
||||
await new Promise(resolve => setTimeout(resolve, 250));
|
||||
if (wrapper.connected) wrapper.send("finish");
|
||||
await exited;
|
||||
await Promise.all([closed, disconnected]);
|
||||
await cancellation;
|
||||
expect(messages, wrapperErrors).toContainEqual({ message: "close-complete", pid: workerPid });
|
||||
expect(messages).toContainEqual({ exitCode: 0, signalCode: null });
|
||||
const diagnostics = `${wrapperErrors}\nwrapper exit=${wrapper.exitCode} signal=${wrapper.signalCode} connected=${wrapper.connected}`;
|
||||
expect(messages, diagnostics).toContainEqual({ message: "close-complete", pid: workerPid });
|
||||
expectSuccessfulWrapperReceipt(wrapper, messages, diagnostics);
|
||||
expect(() => process.kill(workerPid!, 0)).toThrow();
|
||||
} finally {
|
||||
owner.stopObserving();
|
||||
@@ -169,6 +177,21 @@ it.skipIf(process.platform === "win32")("lets launcher cancellation join wrapper
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["failed", "missing"] as const)("rejects a %s worker receipt even when the wrapper closes successfully", async receipt => {
|
||||
const { entry } = await fixture(receipt === "failed"
|
||||
? "process.send({ exitCode: 23, signalCode: null }, () => process.exit(0));"
|
||||
: "process.exit(0);");
|
||||
const wrapper = start(entry);
|
||||
const messages: unknown[] = [];
|
||||
wrapper.on("message", message => messages.push(message));
|
||||
const closed = once(wrapper, "close");
|
||||
const disconnected = once(wrapper, "disconnect");
|
||||
await Promise.all([closed, disconnected]);
|
||||
expect([wrapper.exitCode, wrapper.signalCode]).toEqual([0, null]);
|
||||
expect(messages).toEqual(receipt === "failed" ? [{ exitCode: 23, signalCode: null }] : []);
|
||||
expect(() => expectSuccessfulWrapperReceipt(wrapper, messages)).toThrow();
|
||||
});
|
||||
|
||||
|
||||
it.skipIf(process.platform === "win32")("escalates the owned server group so a hung descendant cannot remain", async () => {
|
||||
const { entry } = await fixture(`
|
||||
@@ -230,6 +253,8 @@ it.skipIf(process.platform === "win32")("recovers a failed cleanup without repea
|
||||
const child = start(entry);
|
||||
await once(child, "message");
|
||||
let failInspection = false;
|
||||
let markInspectionFailed!: () => void;
|
||||
const inspectionFailed = new Promise<void>(resolve => { markInspectionFailed = resolve; });
|
||||
let owner: ReturnType<typeof createProcessTreeOwner> | undefined;
|
||||
const messages: unknown[] = [];
|
||||
child.on("message", message => {
|
||||
@@ -242,14 +267,20 @@ it.skipIf(process.platform === "win32")("recovers a failed cleanup without repea
|
||||
createOwner: candidate => {
|
||||
owner = createProcessTreeOwner(candidate);
|
||||
return { ...owner, observe: async () => {
|
||||
if (failInspection) { failInspection = false; throw new Error("transient inspection failure"); }
|
||||
if (failInspection) {
|
||||
failInspection = false;
|
||||
markInspectionFailed();
|
||||
throw new Error("transient inspection failure");
|
||||
}
|
||||
await owner!.observe();
|
||||
} };
|
||||
},
|
||||
});
|
||||
try {
|
||||
const failed = expect(stop(child)).rejects.toThrow("transient inspection failure");
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
// Hold the child until the one-shot failure is actually injected. A fixed
|
||||
// delay can let it exit during the preceding successful inspection.
|
||||
await inspectionFailed;
|
||||
if (child.connected) child.send("finish");
|
||||
await failed;
|
||||
const recovery = stop(child);
|
||||
|
||||
Reference in new issue
Block a user