From 865d1a72344f36a307a4582d4b0db1fc85ef1934 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 10 Aug 2026 12:51:11 +0000 Subject: [PATCH] fix(runner): close Phase 7 security gate Co-Authored-By: Paperclip --- .../docs/phase-07-eval-conformance.md | 8 + packages/paperclip-runner/package.json | 1 + .../scripts/run-phase7-live-eval-matrix.mjs | 27 ++ .../src/conformance/phase7-eval-suite.test.ts | 10 +- .../src/conformance/phase7-eval-suite.ts | 111 ++++- .../src/phase7/phase7i-demo-server.test.ts | 399 +++++++++++++++++- .../src/phase7/phase7i-demo-server.ts | 16 +- .../src/tools/phase7-semantic-tools.test.ts | 50 ++- 8 files changed, 597 insertions(+), 25 deletions(-) create mode 100644 packages/paperclip-runner/scripts/run-phase7-live-eval-matrix.mjs diff --git a/packages/paperclip-runner/docs/phase-07-eval-conformance.md b/packages/paperclip-runner/docs/phase-07-eval-conformance.md index 7a10c00db1c..03550ceb4ba 100644 --- a/packages/paperclip-runner/docs/phase-07-eval-conformance.md +++ b/packages/paperclip-runner/docs/phase-07-eval-conformance.md @@ -69,6 +69,14 @@ pnpm --filter @paperclipai/paperclip-runner report:phase7-evals ``` The reporter writes `knowledge/evidence/phase-07/eval-parity-report.{json,md}`. + +Run the bounded provider conformance matrix separately. It creates exactly one +real Codex turn for each of the 16 checked-in eval groups while retaining the +in-process mock control plane: + +```sh +pnpm --filter @paperclipai/paperclip-runner report:phase7-live-evals +``` Each failure carries its case ID, assertion class, semantic operation, authorization decision, and final state diff. The report is generated on demand and is not committed; delete it before running `docs:validate` (it carries no diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index efc437dc902..9d1336d6f5e 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -63,6 +63,7 @@ "test:phase7-inventory": "node scripts/check-phase7-inventory.test.mjs", "test:phase7-evals": "vitest run src/conformance/phase7-eval-suite.test.ts", "report:phase7-evals": "pnpm run build:typescript && node scripts/run-phase7-eval-suite.mjs", + "report:phase7-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-phase7-live-eval-matrix.mjs", "check:phase0-parity": "vitest run src/tracer/phase0-runner.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output && node -e \"process.stdout.write('Rust and TypeScript Phase 0 tracer output matches the shared golden fixture.\\n')\"", "check:phase1-parity": "vitest run src/reducer/session-reducer.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core phase1_fixture_parity && node -e \"process.stdout.write('Rust and TypeScript Phase 1 reducer summaries match the shared golden fixtures.\\n')\"", "docs:validate": "node scripts/validate-doc-links.mjs && node scripts/validate-okf.mjs", diff --git a/packages/paperclip-runner/scripts/run-phase7-live-eval-matrix.mjs b/packages/paperclip-runner/scripts/run-phase7-live-eval-matrix.mjs new file mode 100644 index 00000000000..e4fff157931 --- /dev/null +++ b/packages/paperclip-runner/scripts/run-phase7-live-eval-matrix.mjs @@ -0,0 +1,27 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +const packageRoot = resolve(import.meta.dirname, ".."); +const suite = await import(resolve(packageRoot, "dist/conformance/phase7-eval-suite.js")); +const matrix = await suite.runPhase7LiveCodexMatrix(process.cwd()); +const report = { + schema: "paperclip.phase7.live-codex-matrix.v1", + groups: matrix.length, + matrix, +}; +const output = resolve(packageRoot, "knowledge/evidence/phase-07/live-codex-matrix.json"); +const markdown = output.replace(/\.json$/, ".md"); +await mkdir(dirname(output), { recursive: true }); +await Promise.all([ + writeFile(output, `${JSON.stringify(report, null, 2)}\n`), + writeFile(markdown, [ + "# Phase 7 Live Codex Matrix", + "", + `- Groups: ${report.groups}`, + `- Cases: ${matrix.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`, + "- Each row asserts expected and observed typed calls plus final mock state.", + "", + ].join("\n")), +]); +process.stdout.write(`Phase 7 live Codex matrix passed: ${matrix.length} groups.\n`); +process.stdout.write(`Live matrix reports: ${output} and ${markdown}\n`); diff --git a/packages/paperclip-runner/src/conformance/phase7-eval-suite.test.ts b/packages/paperclip-runner/src/conformance/phase7-eval-suite.test.ts index 1fc4f4a8bc2..b7a3d2ad256 100644 --- a/packages/paperclip-runner/src/conformance/phase7-eval-suite.test.ts +++ b/packages/paperclip-runner/src/conformance/phase7-eval-suite.test.ts @@ -15,7 +15,13 @@ describe("Phase 7 offline eval conformance", () => { "restraint_no_call", ]); expect(report.fakeAgentOperationCount).toBe(report.codexOperationCount); - expect(report.boundedCodexSample).toHaveLength(9); - expect(report.results.every((result) => result.caseId && result.semanticOperation && result.authorizationDecision)).toBe(true); + expect(report.boundedCodexSample).toHaveLength(16); + expect(report.results.every((result) => + result.caseId && + result.semanticOperation && + result.authorizationDecision && + result.expectedSemantics.length > 0 && + result.finalState.expected === result.finalState.observed, + )).toBe(true); }); }); diff --git a/packages/paperclip-runner/src/conformance/phase7-eval-suite.ts b/packages/paperclip-runner/src/conformance/phase7-eval-suite.ts index 15dc4c36937..8e15f562fc3 100644 --- a/packages/paperclip-runner/src/conformance/phase7-eval-suite.ts +++ b/packages/paperclip-runner/src/conformance/phase7-eval-suite.ts @@ -6,6 +6,7 @@ import type { Phase7FixtureSeed, Phase7JsonValue } from "../mock-core/phase7-con import { Phase7SemanticToolRuntime } from "../tools/phase7-semantic-tool-runtime.js"; import { Phase7CodexToolBinding, Phase7FakeAgentToolBinding } from "../tools/phase7-tool-bindings.js"; import { phase7FixtureRunCapabilities } from "../phase7/fixture-run-capabilities.js"; +import { Phase7LiveSessionService } from "../phase7/live-session.js"; const GENERATED_HEADER = "# GENERATED FILE — DO NOT EDIT. Run pnpm generate:phase7-inventory.\n"; const PACKAGE_ROOT = resolve(import.meta.dirname, "../.."); @@ -41,11 +42,24 @@ export interface Phase7EvalCaseResult { group: string; assertionClasses: AssertionClass[]; semanticOperation: string; + expectedSemantics: string[]; + forbiddenSemantics: string[]; authorizationDecision: string; stateDiff: string[]; + finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" }; sourceAnchor: string; } +export interface Phase7LiveCodexMatrixResult { + caseId: string; + group: string; + semanticOperation: string; + expectedCalls: string[]; + forbiddenCalls: string[]; + observedCalls: string[]; + finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" }; +} + export interface Phase7EvalParityReport { schema: "paperclip.phase7.eval-parity-report.v1"; cases: number; @@ -161,6 +175,7 @@ async function runCase(row: Phase7EvalRow): Promise { } const stateDiff = before === after ? [] : ["mock_state.revision"]; + const expectedState = operationMutatesState(semanticOperation) ? "mutated" : "unchanged"; if (row.assertionClasses.includes("restraint_no_call") && stateDiff.length !== 0) { throw failure(row, semanticOperation, authorizationDecision, stateDiff); } @@ -170,8 +185,14 @@ async function runCase(row: Phase7EvalRow): Promise { group: row.group, assertionClasses: row.assertionClasses, semanticOperation, + expectedSemantics: row.expectedSemantics, + forbiddenSemantics: row.forbiddenSemantics, authorizationDecision, stateDiff, + finalState: { + expected: expectedState, + observed: stateDiff.length === 0 ? "unchanged" : "mutated", + }, sourceAnchor: row.sourceAnchor, }; } @@ -191,7 +212,7 @@ function operationPlan(row: Phase7EvalRow): OperationPlan { bl: { operationId: "block_task", input: { reason: marker } }, dp: { operationId: "write_document", input: { key: `plan-${row.id}`, title: row.title, body: marker, baseRevisionId: null } }, ix: { operationId: "request_human_input", input: { interactionKind: "confirmation", title: row.title, prompt: marker, continuationPolicy: "wake_assignee" } }, - ar: { operationId: "register_deliverable", input: { filename: `${row.id}.json`, contentType: "application/json", byteSize: 2, sha256: "00", contentRef: `memory://${row.id}`, title: row.title } }, + ar: { operationId: "register_deliverable", input: { filename: `${row.id}.json`, contentType: "application/json", byteSize: 2, sha256: "0".repeat(64), contentRef: `memory://${row.id}`, title: row.title } }, }; const always = alwaysByGroup[row.group]; if (always) return { ...always, grants: [] }; @@ -240,7 +261,7 @@ async function runtimeFor(actorGrants: string[] = []) { } function selectCodexSample(results: Phase7EvalCaseResult[]): Array<{ caseId: string; group: string; semanticOperation: string }> { - const groups = ["hb", "dp", "bl", "ap", "ar", "ix", "mh", "rs", "wk"]; + const groups = [...new Set(results.map((result) => result.group))].sort(); return groups.map((group) => { const result = results.find((candidate) => candidate.group === group); if (!result) throw new Error(`missing bounded Codex sample group ${group}`); @@ -248,6 +269,88 @@ function selectCodexSample(results: Phase7EvalCaseResult[]): Array<{ caseId: str }); } +/** Runs one bounded, real-Codex turn for each checked-in eval group. */ +export async function runPhase7LiveCodexMatrix( + workingDirectory = process.cwd(), +): Promise { + const inventory = await readInventory(); + assertInventory(inventory); + const matrix: Phase7LiveCodexMatrixResult[] = []; + + for (const group of [...new Set(inventory.rows.map((row) => row.group))].sort()) { + const row = inventory.rows.find((candidate) => candidate.group === group); + if (!row) throw new Error(`missing live Codex representative for group ${group}`); + const plan = operationPlan(row); + const service = new Phase7LiveSessionService(); + const session = await service.create({ + workingDirectory, + seed: liveFixtureSeed(plan.grants), + capabilities: phase7FixtureRunCapabilities(plan.grants), + explicitClaims: plan.grants, + turnTimeoutMs: 60_000, + scenario: { id: `phase7-live-eval-${row.id}` }, + }); + try { + const initialMockState = session.snapshot().mockState; + const expectedCalls = row.primaryDisposition === "control_plane_owned" ? [] : [plan.operationId]; + const liveInput = { + ...(typeof plan.input === "object" && plan.input !== null && !Array.isArray(plan.input) ? plan.input : {}), + idempotencyKey: `phase7-live:${row.id}`, + }; + const prompt = expectedCalls.length === 0 + ? "Do not call any tools. Reply with exactly: control-plane-owned." + : `Call ${plan.operationId} exactly once with this JSON input: ${JSON.stringify(liveInput)}. Do not call any other tool. Reply after the typed result.`; + await session.sendMessage(prompt); + const snapshot = session.snapshot(); + const observedCalls = snapshot.evidence + .filter((entry) => entry.kind === "tool_call") + .map((entry) => String(entry.data.operationId)); + const expectedState = operationMutatesState(plan.operationId) ? "mutated" : "unchanged"; + const observedState = snapshot.mockState === initialMockState ? "unchanged" : "mutated"; + if (JSON.stringify(observedCalls) !== JSON.stringify(expectedCalls) || observedState !== expectedState) { + throw new Error(JSON.stringify({ + caseId: row.id, + expectedCalls, + observedCalls, + expectedState, + observedState, + toolResults: snapshot.evidence.filter((entry) => entry.kind === "tool_result").map((entry) => entry.data.result), + })); + } + matrix.push({ + caseId: row.id, + group, + semanticOperation: plan.operationId, + expectedCalls, + forbiddenCalls: row.forbiddenSemantics, + observedCalls, + finalState: { expected: expectedState, observed: observedState }, + }); + } finally { + await service.shutdown(session.id, `Phase 7 live eval ${row.id} complete`); + } + } + return matrix; +} + +function liveFixtureSeed(actorGrants: string[]): Phase7FixtureSeed { + return { + actors: [{ + id: "actor-1", + companyId: "company-1", + name: "Eval fixture actor", + role: "engineer", + status: "active", + budgetId: "budget-actor-1", + capabilityGrants: actorGrants, + }], + }; +} + +function operationMutatesState(operationId: string): boolean { + return !["checkout_task", "search_tasks", "list_agents"].includes(operationId); +} + async function toolSurfaceParity(sample: Array<{ caseId: string; group: string; semanticOperation: string }>) { const { runtime } = await runtimeFor([ "discovery:tasks:read", @@ -290,13 +393,13 @@ function renderMarkdown(report: Phase7EvalParityReport): string { `- Groups: ${report.groups.length} (${report.groups.join(", ")})`, `- Assertion classes: ${report.assertionClasses.join(", ")}`, `- Fake-agent/Codex operations: ${report.fakeAgentOperationCount}/${report.codexOperationCount}`, - `- Bounded Codex binding sample: ${report.boundedCodexSample.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`, + `- Bounded Codex binding matrix: ${report.boundedCodexSample.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`, `- Semantic execution: ${Object.entries(byDisposition).map(([operation, count]) => `${operation}=${count}`).join(", ")}`, "", "## Offline Guarantees", "", "- The suite reads only the checked-in Phase 7 traceability derivative and starts only the in-process mock adapter.", - "- Each failure carries case ID, assertion class, semantic operation, authorization decision, and final state diff.", + "- Each result includes source expected/forbidden semantics plus expected and observed final mock state.", "- Optional operations are evaluated in both granted and absent configurations; control-plane-owned operations remain absent.", "", ].join("\n"); diff --git a/packages/paperclip-runner/src/phase7/phase7i-demo-server.test.ts b/packages/paperclip-runner/src/phase7/phase7i-demo-server.test.ts index 01e38c754be..7954d4f91f8 100644 --- a/packages/paperclip-runner/src/phase7/phase7i-demo-server.test.ts +++ b/packages/paperclip-runner/src/phase7/phase7i-demo-server.test.ts @@ -1,8 +1,10 @@ -import { readFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { request as httpRequest } from "node:http"; +import { tmpdir } from "node:os"; import { dirname, extname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { assertPhase7iEnvironmentSafe, @@ -10,6 +12,7 @@ import { phase7iCapabilityCookieForOrigin, Phase7iDemoRuntime, } from "./phase7i-demo-server.js"; +import { Phase7ChatSession } from "./chat-session.js"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const sourceRoot = resolve(packageRoot, "src"); @@ -105,8 +108,400 @@ describe("Phase 7I deployment boundary", () => { } } }); + + it("requires the trusted proxy marker and exact forwarded authority over HTTP", async () => { + await withHttpRuntime(async ({ request }) => { + expect(await request("/api/phase7/health", { + headers: { ...proxyHeaders(), "x-phase7i-proxy": "" }, + })).toMatchObject({ + status: 403, + body: { error: { code: "trusted_proxy_required" } }, + }); + expect(await request("/api/phase7/health", { + headers: { ...proxyHeaders(), "x-forwarded-proto": "https,http" }, + })).toMatchObject({ status: 403, body: { error: { code: "transport_denied" } } }); + expect(await request("/api/phase7/health", { + headers: { + ...proxyHeaders(), + "x-forwarded-host": "phase7i.example.ts.net,attacker.example", + }, + })).toMatchObject({ status: 403, body: { error: { code: "host_denied" } } }); + expect(await request("/api/phase7/health", { headers: proxyHeaders() })).toMatchObject({ + status: 200, + body: { status: "ok", controlPlane: "phase7-mock" }, + }); + }); + }); + + it("requires exact Origin and Fetch Metadata on every mutation", async () => { + await withHttpRuntime(async ({ request }) => { + const body = JSON.stringify({ scenarioId: SCENARIO_ID }); + const invalidHeaders = [ + { origin: `${PUBLIC_ORIGIN}/` }, + { "sec-fetch-site": "none" }, + { "sec-fetch-site": "cross-site" }, + { "sec-fetch-mode": "same-origin" }, + { "sec-fetch-dest": "document" }, + ]; + for (const replacement of invalidHeaders) { + const response = await request("/api/phase7/sessions", { + method: "POST", + headers: { ...mutationHeaders(), ...replacement }, + body, + }); + expect(response.status).toBe(403); + expect(response.body).toMatchObject({ error: { code: expect.stringMatching(/^(?:origin|fetch_metadata)_denied$/) } }); + } + const missingFetchSite = mutationHeaders(); + delete missingFetchSite["sec-fetch-site"]; + expect(await request("/api/phase7/sessions", { + method: "POST", + headers: missingFetchSite, + body, + })).toMatchObject({ status: 403, body: { error: { code: "fetch_metadata_denied" } } }); + }); + }); + + it("accepts only JSON objects with the exact route fields", async () => { + await withHttpRuntime(async ({ request }) => { + for (const contentType of ["text/plain", "application/jsonp"]) { + expect(await request("/api/phase7/sessions", { + method: "POST", + headers: { ...mutationHeaders(), "content-type": contentType }, + body: JSON.stringify({ scenarioId: SCENARIO_ID }), + })).toMatchObject({ status: 415, body: { error: { code: "json_required" } } }); + } + expect(await request("/api/phase7/sessions", { + method: "POST", + headers: mutationHeaders(), + body: JSON.stringify({ scenarioId: SCENARIO_ID, adapter: "paperclip" }), + })).toMatchObject({ status: 422, body: { error: { code: "unexpected_field" } } }); + }); + }); + + it("returns the same 404 for missing, foreign, or identity-mismatched session authority", async () => { + await withHttpRuntime(async ({ request }) => { + const created = await createSession(request, "alice@example.com"); + const failures = [ + mutationHeaders("alice@example.com"), + { ...mutationHeaders("alice@example.com"), cookie: "__Host-paperclip_phase7i=foreign" }, + { ...mutationHeaders("mallory@example.com"), cookie: created.cookie }, + ]; + for (const headers of failures) { + const response = await request(`/api/phase7/sessions/${created.id}/replay`, { + method: "POST", + headers, + body: "{}", + }); + expect(response).toMatchObject({ + status: 404, + body: { error: { code: "session_not_found", message: "Session was not found." } }, + }); + } + }); + }); + + it("isolates two sessions even when the tailnet identity is shared", async () => { + await withHttpRuntime(async ({ request }) => { + const first = await createSession(request, "alice@example.com"); + const second = await createSession(request, "alice@example.com"); + expect(first.id).not.toBe(second.id); + expect(first.cookie).not.toBe(second.cookie); + + expect(await replaySession(request, second.id, first.cookie, "alice@example.com")).toMatchObject({ + status: 404, + body: { error: { code: "session_not_found" } }, + }); + expect(await replaySession(request, first.id, first.cookie, "alice@example.com")).toMatchObject({ + status: 200, + body: { sessionId: first.id }, + }); + expect(await replaySession(request, second.id, second.cookie, "alice@example.com")).toMatchObject({ + status: 200, + body: { sessionId: second.id }, + }); + }); + }); + + it("rotates both session id and capability on reset and invalidates the old pair", async () => { + await withHttpRuntime(async ({ request }) => { + const original = await createSession(request, "alice@example.com"); + const reset = await request(`/api/phase7/sessions/${original.id}/reset`, { + method: "POST", + headers: { ...mutationHeaders("alice@example.com"), cookie: original.cookie }, + body: "{}", + }); + expect(reset.status).toBe(200); + const replacementId = sessionId(reset); + const replacementCookie = responseCookie(reset); + expect(replacementId).not.toBe(original.id); + expect(replacementCookie).not.toBe(original.cookie); + + expect(await replaySession(request, original.id, original.cookie, "alice@example.com")).toMatchObject({ + status: 404, + body: { error: { code: "session_not_found" } }, + }); + expect(await replaySession(request, replacementId, original.cookie, "alice@example.com")).toMatchObject({ + status: 404, + body: { error: { code: "session_not_found" } }, + }); + expect(await replaySession(request, replacementId, replacementCookie, "alice@example.com")).toMatchObject({ + status: 200, + body: { sessionId: replacementId }, + }); + }); + }); + + it("denies a concurrent second turn without invoking the chat twice", async () => { + let release!: () => void; + let entered!: () => void; + const gate = new Promise((resolveGate) => { + release = resolveGate; + }); + const started = new Promise((resolveStarted) => { + entered = resolveStarted; + }); + const originalSend = Phase7ChatSession.prototype.send; + const send = vi.spyOn(Phase7ChatSession.prototype, "send").mockImplementationOnce( + async function (this: Phase7ChatSession, prompt: string) { + entered(); + await gate; + return await originalSend.call(this, prompt); + }, + ); + try { + await withHttpRuntime(async ({ request }) => { + const session = await createSession(request, "alice@example.com"); + const firstTurn = request(`/api/phase7/sessions/${session.id}/turn`, { + method: "POST", + headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie }, + body: JSON.stringify({ prompt: "First turn" }), + }); + await started; + expect(await request(`/api/phase7/sessions/${session.id}/turn`, { + method: "POST", + headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie }, + body: JSON.stringify({ prompt: "Second turn" }), + })).toMatchObject({ status: 409, body: { error: { code: "turn_active" } } }); + release(); + expect(await firstTurn).toMatchObject({ status: 200, body: { sessionId: session.id } }); + expect(send).toHaveBeenCalledTimes(1); + }); + } finally { + release(); + send.mockRestore(); + } + }); + + it("enforces prompt, body, session-capacity, and request-rate bounds", async () => { + await withHttpRuntime(async ({ request }) => { + const session = await createSession(request, "alice@example.com"); + expect(await request(`/api/phase7/sessions/${session.id}/turn`, { + method: "POST", + headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie }, + body: JSON.stringify({ prompt: "x".repeat(16 * 1024 + 1) }), + })).toMatchObject({ status: 422, body: { error: { code: "prompt_too_large" } } }); + expect(await request("/api/phase7/sessions", { + method: "POST", + headers: mutationHeaders("body@example.com"), + body: JSON.stringify({ scenarioId: SCENARIO_ID, padding: "x".repeat(64 * 1024) }), + })).toMatchObject({ status: 413, body: { error: { code: "request_too_large" } } }); + + for (let index = 1; index < 8; index += 1) { + expect((await createSession(request, `capacity-${index}@example.com`)).id).toHaveLength(32); + } + expect(await request("/api/phase7/sessions", { + method: "POST", + headers: mutationHeaders("capacity-overflow@example.com"), + body: JSON.stringify({ scenarioId: SCENARIO_ID }), + })).toMatchObject({ status: 429, body: { error: { code: "session_capacity" } } }); + }); + + await withHttpRuntime(async ({ request }) => { + for (let index = 0; index < 90; index += 1) { + expect((await request(`/api/phase7/missing-${index}`, { + method: "POST", + headers: mutationHeaders("rate@example.com"), + body: "{}", + })).status).toBe(404); + } + expect(await request("/api/phase7/rate-limit", { + method: "POST", + headers: mutationHeaders("rate@example.com"), + body: "{}", + })).toMatchObject({ status: 429, body: { error: { code: "rate_limited" } } }); + }); + }); + + it("returns stable redacted internal errors and security headers", async () => { + await withHttpRuntime(async ({ request }) => { + const first = await request("/%", { headers: proxyHeaders() }); + const second = await request("/%E0%A4%A", { headers: proxyHeaders() }); + expect(first).toMatchObject({ + status: 500, + body: { error: { code: "internal_error", message: "The request could not be completed." } }, + }); + expect(second.body).toEqual(first.body); + expect(JSON.stringify(first.body)).not.toMatch(/URIError|decodeURIComponent|%E0|stack/i); + + for (const response of [first, await request("/api/phase7/health", { headers: proxyHeaders() })]) { + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(response.headers.get("content-security-policy")).toContain("default-src 'none'"); + expect(response.headers.get("cross-origin-opener-policy")).toBe("same-origin"); + expect(response.headers.get("cross-origin-resource-policy")).toBe("same-origin"); + expect(response.headers.get("permissions-policy")).toContain("camera=()"); + expect(response.headers.get("referrer-policy")).toBe("no-referrer"); + expect(response.headers.get("strict-transport-security")).toBe("max-age=31536000"); + expect(response.headers.get("x-content-type-options")).toBe("nosniff"); + expect(response.headers.get("x-frame-options")).toBe("DENY"); + } + }); + }); }); +const PUBLIC_ORIGIN = "https://phase7i.example.ts.net"; +const SCENARIO_ID = "hb-inbox-lite-01"; + +type HttpResult = { + status: number; + headers: Headers; + body: unknown; +}; + +type HttpRequest = ( + path: string, + init?: { method?: string; headers?: Record; body?: string }, +) => Promise; + +function proxyHeaders(identity = "alice@example.com"): Record { + return { + "tailscale-user-login": identity, + "x-forwarded-host": new URL(PUBLIC_ORIGIN).host, + "x-forwarded-proto": new URL(PUBLIC_ORIGIN).protocol.slice(0, -1), + "x-phase7i-proxy": "v1", + }; +} + +function mutationHeaders(identity = "alice@example.com"): Record { + return { + ...proxyHeaders(identity), + "content-type": "application/json; charset=utf-8", + origin: PUBLIC_ORIGIN, + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + }; +} + +async function withHttpRuntime( + run: (context: { runtime: Phase7iDemoRuntime; request: HttpRequest }) => Promise, +): Promise { + const runtime = new Phase7iDemoRuntime({ + publicOrigin: PUBLIC_ORIGIN, + basePath: "", + commitSha: "deadbeef", + packageRoot, + log: () => undefined, + }); + await runtime.initialize(); + const server = runtime.createServer(); + const scratchRoot = process.env.PAPERCLIP_RUN_SCRATCH_DIR ?? tmpdir(); + const socketDirectory = await mkdtemp(resolve(scratchRoot, "phase7i-http-")); + const socketPath = resolve(socketDirectory, "server.sock"); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen(socketPath, () => { + server.off("error", rejectListen); + resolveListen(); + }); + }); + const request: HttpRequest = async (path, init = {}) => { + return await new Promise((resolveRequest, rejectRequest) => { + const request = httpRequest({ + socketPath, + path, + method: init.method ?? "GET", + headers: init.headers, + }, (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer | string) => { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + response.once("error", rejectRequest); + response.once("end", () => { + const headers = new Headers(); + for (const [name, value] of Object.entries(response.headers)) { + if (Array.isArray(value)) { + for (const item of value) headers.append(name, item); + } else if (value !== undefined) { + headers.set(name, String(value)); + } + } + const text = Buffer.concat(chunks).toString("utf8"); + resolveRequest({ + status: response.statusCode ?? 0, + headers, + body: text === "" ? null : JSON.parse(text) as unknown, + }); + }); + }); + request.once("error", rejectRequest); + if (init.body !== undefined) request.write(init.body); + request.end(); + }); + }; + try { + await run({ runtime, request }); + } finally { + await new Promise((resolveClose, rejectClose) => { + server.close((error) => error ? rejectClose(error) : resolveClose()); + }); + await runtime.shutdown(); + await rm(socketDirectory, { recursive: true, force: true }); + } +} + +async function createSession(request: HttpRequest, identity: string): Promise<{ id: string; cookie: string }> { + const response = await request("/api/phase7/sessions", { + method: "POST", + headers: mutationHeaders(identity), + body: JSON.stringify({ scenarioId: SCENARIO_ID }), + }); + expect(response.status).toBe(201); + return { id: sessionId(response), cookie: responseCookie(response) }; +} + +function sessionId(response: HttpResult): string { + if ( + typeof response.body !== "object" || + response.body === null || + !("sessionId" in response.body) || + typeof response.body.sessionId !== "string" + ) { + throw new Error("response did not contain a session id"); + } + return response.body.sessionId; +} + +function responseCookie(response: HttpResult): string { + const setCookie = response.headers.get("set-cookie"); + if (!setCookie) throw new Error("response did not set a capability cookie"); + return setCookie.split(";", 1)[0]!; +} + +async function replaySession( + request: HttpRequest, + id: string, + cookie: string, + identity: string, +): Promise { + return await request(`/api/phase7/sessions/${id}/replay`, { + method: "POST", + headers: { ...mutationHeaders(identity), cookie }, + body: "{}", + }); +} + function importSpecifiers(source: string): string[] { const matches = source.matchAll(/\b(?:import|export)\s+(?:type\s+)?(?:[^"'`;]{0,500}?\s+from\s+)?["']([^"']+)["']/g); return [...matches].map((match) => match[1]!); diff --git a/packages/paperclip-runner/src/phase7/phase7i-demo-server.ts b/packages/paperclip-runner/src/phase7/phase7i-demo-server.ts index 14f7415f876..82c2fc7683d 100644 --- a/packages/paperclip-runner/src/phase7/phase7i-demo-server.ts +++ b/packages/paperclip-runner/src/phase7/phase7i-demo-server.ts @@ -159,7 +159,8 @@ function sendError(res: ServerResponse, error: RequestFailure): void { } async function readJsonBody(req: IncomingMessage): Promise> { - if (!header(req, "content-type").toLowerCase().startsWith("application/json")) { + const mediaType = header(req, "content-type").split(";", 1)[0]?.trim().toLowerCase(); + if (mediaType !== "application/json") { throw new RequestFailure(415, "json_required", "Mutation requests require application/json."); } const declaredLength = Number(header(req, "content-length")); @@ -359,11 +360,11 @@ export class Phase7iDemoRuntime { if (header(req, "x-phase7i-proxy") !== "v1") { throw new RequestFailure(403, "trusted_proxy_required", "Request did not arrive through the trusted proxy."); } - const forwardedProtocol = header(req, "x-forwarded-proto").split(",")[0]?.trim().toLowerCase(); + const forwardedProtocol = header(req, "x-forwarded-proto").toLowerCase(); if (forwardedProtocol !== this.publicOrigin.protocol.slice(0, -1)) { throw new RequestFailure(403, "transport_denied", "Forwarded transport is not allowed."); } - const forwardedHost = header(req, "x-forwarded-host").split(",")[0]?.trim().toLowerCase(); + const forwardedHost = header(req, "x-forwarded-host").toLowerCase(); if (forwardedHost !== this.publicOrigin.host.toLowerCase()) { throw new RequestFailure(403, "host_denied", "Forwarded host is not allowed."); } @@ -374,9 +375,12 @@ export class Phase7iDemoRuntime { if (header(req, "origin") !== this.publicOrigin.origin) { throw new RequestFailure(403, "origin_denied", "Request origin is not allowed."); } - const fetchSite = header(req, "sec-fetch-site").toLowerCase(); - if (fetchSite && fetchSite !== "same-origin" && fetchSite !== "none") { - throw new RequestFailure(403, "fetch_metadata_denied", "Cross-site mutation requests are not allowed."); + if ( + header(req, "sec-fetch-site").toLowerCase() !== "same-origin" || + header(req, "sec-fetch-mode").toLowerCase() !== "cors" || + header(req, "sec-fetch-dest").toLowerCase() !== "empty" + ) { + throw new RequestFailure(403, "fetch_metadata_denied", "Mutation request metadata is not allowed."); } } diff --git a/packages/paperclip-runner/src/tools/phase7-semantic-tools.test.ts b/packages/paperclip-runner/src/tools/phase7-semantic-tools.test.ts index fd06384cc76..f2be30289c3 100644 --- a/packages/paperclip-runner/src/tools/phase7-semantic-tools.test.ts +++ b/packages/paperclip-runner/src/tools/phase7-semantic-tools.test.ts @@ -257,17 +257,6 @@ describe("Phase 7 exposure and authorization", () => { }); expect(revisions).toMatchObject({ ok: true, value: [{ id: revisionId, revision: 1 }] }); - expect(await runtime.invoke({ - operationId: "request_human_input", - input: { - interactionKind: "confirmation", - title: "Confirm plan", - prompt: "Accept this plan?", - targetRevisionId: revisionId!, - continuationPolicy: "wake_assignee", - }, - idempotencyKey: "confirm-plan-1", - })).toMatchObject({ ok: true }); expect(await runtime.invoke({ operationId: "register_deliverable", input: { @@ -285,6 +274,17 @@ describe("Phase 7 exposure and authorization", () => { input: { summary: "Ready for semantic review." }, idempotencyKey: "review-1", })).toMatchObject({ ok: true }); + expect(await runtime.invoke({ + operationId: "request_human_input", + input: { + interactionKind: "confirmation", + title: "Confirm plan", + prompt: "Accept this plan?", + targetRevisionId: revisionId!, + continuationPolicy: "wake_assignee", + }, + idempotencyKey: "confirm-plan-1", + })).toMatchObject({ ok: true }); expect(adapter.snapshot()).toMatchObject({ tasks: [{ status: "in_review" }], @@ -294,6 +294,34 @@ describe("Phase 7 exposure and authorization", () => { workProducts: [{ type: "artifact" }], }); }); + + it("keeps a pending human-input handoff in review instead of weakening the lifecycle", async () => { + const { adapter, runtime } = await runtimeFor(); + expect(await runtime.invoke({ + operationId: "request_human_input", + input: { + interactionKind: "confirmation", + title: "Confirm plan", + prompt: "Accept this plan?", + continuationPolicy: "wake_assignee", + }, + idempotencyKey: "confirm-plan-first", + })).toMatchObject({ ok: true }); + + expect(await runtime.invoke({ + operationId: "request_review", + input: { summary: "A duplicate review transition must fail closed." }, + idempotencyKey: "duplicate-review", + })).toMatchObject({ + ok: false, + error: { code: "operation_unsupported", reason: "mock_operation_rejected" }, + }); + expect(adapter.snapshot()).toMatchObject({ + tasks: [{ status: "in_review" }], + interactions: [{ status: "pending" }], + comments: [], + }); + }); }); describe("Phase 7 security policy", () => {