fix(runner): close Phase 7 security gate

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-10 12:51:11 +00:00
1 parent fe231fbb7e
commit 865d1a7234
8 files changed
+597 -25

No files matched your search

@@ -69,6 +69,14 @@ pnpm --filter @paperclipai/paperclip-runner report:phase7-evals
``` ```
The reporter writes `knowledge/evidence/phase-07/eval-parity-report.{json,md}`. The reporter writes `knowledge/evidence/phase-07/eval-parity-report.{json,md}`.
Run the bounded provider conformance matrix separately. It creates exactly one
real Codex turn for each of the 16 checked-in eval groups while retaining the
in-process mock control plane:
```sh
pnpm --filter @paperclipai/paperclip-runner report:phase7-live-evals
```
Each failure carries its case ID, assertion class, semantic operation, Each failure carries its case ID, assertion class, semantic operation,
authorization decision, and final state diff. The report is generated on demand authorization decision, and final state diff. The report is generated on demand
and is not committed; delete it before running `docs:validate` (it carries no and is not committed; delete it before running `docs:validate` (it carries no
+1
View File
@@ -63,6 +63,7 @@
"test:phase7-inventory": "node scripts/check-phase7-inventory.test.mjs", "test:phase7-inventory": "node scripts/check-phase7-inventory.test.mjs",
"test:phase7-evals": "vitest run src/conformance/phase7-eval-suite.test.ts", "test:phase7-evals": "vitest run src/conformance/phase7-eval-suite.test.ts",
"report:phase7-evals": "pnpm run build:typescript && node scripts/run-phase7-eval-suite.mjs", "report:phase7-evals": "pnpm run build:typescript && node scripts/run-phase7-eval-suite.mjs",
"report:phase7-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-phase7-live-eval-matrix.mjs",
"check:phase0-parity": "vitest run src/tracer/phase0-runner.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output && node -e \"process.stdout.write('Rust and TypeScript Phase 0 tracer output matches the shared golden fixture.\\n')\"", "check:phase0-parity": "vitest run src/tracer/phase0-runner.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output && node -e \"process.stdout.write('Rust and TypeScript Phase 0 tracer output matches the shared golden fixture.\\n')\"",
"check:phase1-parity": "vitest run src/reducer/session-reducer.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core phase1_fixture_parity && node -e \"process.stdout.write('Rust and TypeScript Phase 1 reducer summaries match the shared golden fixtures.\\n')\"", "check:phase1-parity": "vitest run src/reducer/session-reducer.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core phase1_fixture_parity && node -e \"process.stdout.write('Rust and TypeScript Phase 1 reducer summaries match the shared golden fixtures.\\n')\"",
"docs:validate": "node scripts/validate-doc-links.mjs && node scripts/validate-okf.mjs", "docs:validate": "node scripts/validate-doc-links.mjs && node scripts/validate-okf.mjs",
@@ -0,0 +1,27 @@
import { mkdir, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
const packageRoot = resolve(import.meta.dirname, "..");
const suite = await import(resolve(packageRoot, "dist/conformance/phase7-eval-suite.js"));
const matrix = await suite.runPhase7LiveCodexMatrix(process.cwd());
const report = {
schema: "paperclip.phase7.live-codex-matrix.v1",
groups: matrix.length,
matrix,
};
const output = resolve(packageRoot, "knowledge/evidence/phase-07/live-codex-matrix.json");
const markdown = output.replace(/\.json$/, ".md");
await mkdir(dirname(output), { recursive: true });
await Promise.all([
writeFile(output, `${JSON.stringify(report, null, 2)}\n`),
writeFile(markdown, [
"# Phase 7 Live Codex Matrix",
"",
`- Groups: ${report.groups}`,
`- Cases: ${matrix.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`,
"- Each row asserts expected and observed typed calls plus final mock state.",
"",
].join("\n")),
]);
process.stdout.write(`Phase 7 live Codex matrix passed: ${matrix.length} groups.\n`);
process.stdout.write(`Live matrix reports: ${output} and ${markdown}\n`);
@@ -15,7 +15,13 @@ describe("Phase 7 offline eval conformance", () => {
"restraint_no_call", "restraint_no_call",
]); ]);
expect(report.fakeAgentOperationCount).toBe(report.codexOperationCount); expect(report.fakeAgentOperationCount).toBe(report.codexOperationCount);
expect(report.boundedCodexSample).toHaveLength(9); expect(report.boundedCodexSample).toHaveLength(16);
expect(report.results.every((result) => result.caseId && result.semanticOperation && result.authorizationDecision)).toBe(true); expect(report.results.every((result) =>
result.caseId &&
result.semanticOperation &&
result.authorizationDecision &&
result.expectedSemantics.length > 0 &&
result.finalState.expected === result.finalState.observed,
)).toBe(true);
}); });
}); });
@@ -6,6 +6,7 @@ import type { Phase7FixtureSeed, Phase7JsonValue } from "../mock-core/phase7-con
import { Phase7SemanticToolRuntime } from "../tools/phase7-semantic-tool-runtime.js"; import { Phase7SemanticToolRuntime } from "../tools/phase7-semantic-tool-runtime.js";
import { Phase7CodexToolBinding, Phase7FakeAgentToolBinding } from "../tools/phase7-tool-bindings.js"; import { Phase7CodexToolBinding, Phase7FakeAgentToolBinding } from "../tools/phase7-tool-bindings.js";
import { phase7FixtureRunCapabilities } from "../phase7/fixture-run-capabilities.js"; import { phase7FixtureRunCapabilities } from "../phase7/fixture-run-capabilities.js";
import { Phase7LiveSessionService } from "../phase7/live-session.js";
const GENERATED_HEADER = "# GENERATED FILE — DO NOT EDIT. Run pnpm generate:phase7-inventory.\n"; const GENERATED_HEADER = "# GENERATED FILE — DO NOT EDIT. Run pnpm generate:phase7-inventory.\n";
const PACKAGE_ROOT = resolve(import.meta.dirname, "../.."); const PACKAGE_ROOT = resolve(import.meta.dirname, "../..");
@@ -41,11 +42,24 @@ export interface Phase7EvalCaseResult {
group: string; group: string;
assertionClasses: AssertionClass[]; assertionClasses: AssertionClass[];
semanticOperation: string; semanticOperation: string;
expectedSemantics: string[];
forbiddenSemantics: string[];
authorizationDecision: string; authorizationDecision: string;
stateDiff: string[]; stateDiff: string[];
finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" };
sourceAnchor: string; sourceAnchor: string;
} }
export interface Phase7LiveCodexMatrixResult {
caseId: string;
group: string;
semanticOperation: string;
expectedCalls: string[];
forbiddenCalls: string[];
observedCalls: string[];
finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" };
}
export interface Phase7EvalParityReport { export interface Phase7EvalParityReport {
schema: "paperclip.phase7.eval-parity-report.v1"; schema: "paperclip.phase7.eval-parity-report.v1";
cases: number; cases: number;
@@ -161,6 +175,7 @@ async function runCase(row: Phase7EvalRow): Promise<Phase7EvalCaseResult> {
} }
const stateDiff = before === after ? [] : ["mock_state.revision"]; const stateDiff = before === after ? [] : ["mock_state.revision"];
const expectedState = operationMutatesState(semanticOperation) ? "mutated" : "unchanged";
if (row.assertionClasses.includes("restraint_no_call") && stateDiff.length !== 0) { if (row.assertionClasses.includes("restraint_no_call") && stateDiff.length !== 0) {
throw failure(row, semanticOperation, authorizationDecision, stateDiff); throw failure(row, semanticOperation, authorizationDecision, stateDiff);
} }
@@ -170,8 +185,14 @@ async function runCase(row: Phase7EvalRow): Promise<Phase7EvalCaseResult> {
group: row.group, group: row.group,
assertionClasses: row.assertionClasses, assertionClasses: row.assertionClasses,
semanticOperation, semanticOperation,
expectedSemantics: row.expectedSemantics,
forbiddenSemantics: row.forbiddenSemantics,
authorizationDecision, authorizationDecision,
stateDiff, stateDiff,
finalState: {
expected: expectedState,
observed: stateDiff.length === 0 ? "unchanged" : "mutated",
},
sourceAnchor: row.sourceAnchor, sourceAnchor: row.sourceAnchor,
}; };
} }
@@ -191,7 +212,7 @@ function operationPlan(row: Phase7EvalRow): OperationPlan {
bl: { operationId: "block_task", input: { reason: marker } }, bl: { operationId: "block_task", input: { reason: marker } },
dp: { operationId: "write_document", input: { key: `plan-${row.id}`, title: row.title, body: marker, baseRevisionId: null } }, dp: { operationId: "write_document", input: { key: `plan-${row.id}`, title: row.title, body: marker, baseRevisionId: null } },
ix: { operationId: "request_human_input", input: { interactionKind: "confirmation", title: row.title, prompt: marker, continuationPolicy: "wake_assignee" } }, ix: { operationId: "request_human_input", input: { interactionKind: "confirmation", title: row.title, prompt: marker, continuationPolicy: "wake_assignee" } },
ar: { operationId: "register_deliverable", input: { filename: `${row.id}.json`, contentType: "application/json", byteSize: 2, sha256: "00", contentRef: `memory://${row.id}`, title: row.title } }, ar: { operationId: "register_deliverable", input: { filename: `${row.id}.json`, contentType: "application/json", byteSize: 2, sha256: "0".repeat(64), contentRef: `memory://${row.id}`, title: row.title } },
}; };
const always = alwaysByGroup[row.group]; const always = alwaysByGroup[row.group];
if (always) return { ...always, grants: [] }; if (always) return { ...always, grants: [] };
@@ -240,7 +261,7 @@ async function runtimeFor(actorGrants: string[] = []) {
} }
function selectCodexSample(results: Phase7EvalCaseResult[]): Array<{ caseId: string; group: string; semanticOperation: string }> { function selectCodexSample(results: Phase7EvalCaseResult[]): Array<{ caseId: string; group: string; semanticOperation: string }> {
const groups = ["hb", "dp", "bl", "ap", "ar", "ix", "mh", "rs", "wk"]; const groups = [...new Set(results.map((result) => result.group))].sort();
return groups.map((group) => { return groups.map((group) => {
const result = results.find((candidate) => candidate.group === group); const result = results.find((candidate) => candidate.group === group);
if (!result) throw new Error(`missing bounded Codex sample group ${group}`); if (!result) throw new Error(`missing bounded Codex sample group ${group}`);
@@ -248,6 +269,88 @@ function selectCodexSample(results: Phase7EvalCaseResult[]): Array<{ caseId: str
}); });
} }
/** Runs one bounded, real-Codex turn for each checked-in eval group. */
export async function runPhase7LiveCodexMatrix(
workingDirectory = process.cwd(),
): Promise<Phase7LiveCodexMatrixResult[]> {
const inventory = await readInventory();
assertInventory(inventory);
const matrix: Phase7LiveCodexMatrixResult[] = [];
for (const group of [...new Set(inventory.rows.map((row) => row.group))].sort()) {
const row = inventory.rows.find((candidate) => candidate.group === group);
if (!row) throw new Error(`missing live Codex representative for group ${group}`);
const plan = operationPlan(row);
const service = new Phase7LiveSessionService();
const session = await service.create({
workingDirectory,
seed: liveFixtureSeed(plan.grants),
capabilities: phase7FixtureRunCapabilities(plan.grants),
explicitClaims: plan.grants,
turnTimeoutMs: 60_000,
scenario: { id: `phase7-live-eval-${row.id}` },
});
try {
const initialMockState = session.snapshot().mockState;
const expectedCalls = row.primaryDisposition === "control_plane_owned" ? [] : [plan.operationId];
const liveInput = {
...(typeof plan.input === "object" && plan.input !== null && !Array.isArray(plan.input) ? plan.input : {}),
idempotencyKey: `phase7-live:${row.id}`,
};
const prompt = expectedCalls.length === 0
? "Do not call any tools. Reply with exactly: control-plane-owned."
: `Call ${plan.operationId} exactly once with this JSON input: ${JSON.stringify(liveInput)}. Do not call any other tool. Reply after the typed result.`;
await session.sendMessage(prompt);
const snapshot = session.snapshot();
const observedCalls = snapshot.evidence
.filter((entry) => entry.kind === "tool_call")
.map((entry) => String(entry.data.operationId));
const expectedState = operationMutatesState(plan.operationId) ? "mutated" : "unchanged";
const observedState = snapshot.mockState === initialMockState ? "unchanged" : "mutated";
if (JSON.stringify(observedCalls) !== JSON.stringify(expectedCalls) || observedState !== expectedState) {
throw new Error(JSON.stringify({
caseId: row.id,
expectedCalls,
observedCalls,
expectedState,
observedState,
toolResults: snapshot.evidence.filter((entry) => entry.kind === "tool_result").map((entry) => entry.data.result),
}));
}
matrix.push({
caseId: row.id,
group,
semanticOperation: plan.operationId,
expectedCalls,
forbiddenCalls: row.forbiddenSemantics,
observedCalls,
finalState: { expected: expectedState, observed: observedState },
});
} finally {
await service.shutdown(session.id, `Phase 7 live eval ${row.id} complete`);
}
}
return matrix;
}
function liveFixtureSeed(actorGrants: string[]): Phase7FixtureSeed {
return {
actors: [{
id: "actor-1",
companyId: "company-1",
name: "Eval fixture actor",
role: "engineer",
status: "active",
budgetId: "budget-actor-1",
capabilityGrants: actorGrants,
}],
};
}
function operationMutatesState(operationId: string): boolean {
return !["checkout_task", "search_tasks", "list_agents"].includes(operationId);
}
async function toolSurfaceParity(sample: Array<{ caseId: string; group: string; semanticOperation: string }>) { async function toolSurfaceParity(sample: Array<{ caseId: string; group: string; semanticOperation: string }>) {
const { runtime } = await runtimeFor([ const { runtime } = await runtimeFor([
"discovery:tasks:read", "discovery:tasks:read",
@@ -290,13 +393,13 @@ function renderMarkdown(report: Phase7EvalParityReport): string {
`- Groups: ${report.groups.length} (${report.groups.join(", ")})`, `- Groups: ${report.groups.length} (${report.groups.join(", ")})`,
`- Assertion classes: ${report.assertionClasses.join(", ")}`, `- Assertion classes: ${report.assertionClasses.join(", ")}`,
`- Fake-agent/Codex operations: ${report.fakeAgentOperationCount}/${report.codexOperationCount}`, `- Fake-agent/Codex operations: ${report.fakeAgentOperationCount}/${report.codexOperationCount}`,
`- Bounded Codex binding sample: ${report.boundedCodexSample.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`, `- Bounded Codex binding matrix: ${report.boundedCodexSample.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`,
`- Semantic execution: ${Object.entries(byDisposition).map(([operation, count]) => `${operation}=${count}`).join(", ")}`, `- Semantic execution: ${Object.entries(byDisposition).map(([operation, count]) => `${operation}=${count}`).join(", ")}`,
"", "",
"## Offline Guarantees", "## Offline Guarantees",
"", "",
"- The suite reads only the checked-in Phase 7 traceability derivative and starts only the in-process mock adapter.", "- The suite reads only the checked-in Phase 7 traceability derivative and starts only the in-process mock adapter.",
"- Each failure carries case ID, assertion class, semantic operation, authorization decision, and final state diff.", "- Each result includes source expected/forbidden semantics plus expected and observed final mock state.",
"- Optional operations are evaluated in both granted and absent configurations; control-plane-owned operations remain absent.", "- Optional operations are evaluated in both granted and absent configurations; control-plane-owned operations remain absent.",
"", "",
].join("\n"); ].join("\n");
@@ -1,8 +1,10 @@
import { readFile } from "node:fs/promises"; import { mkdtemp, readFile, rm } from "node:fs/promises";
import { request as httpRequest } from "node:http";
import { tmpdir } from "node:os";
import { dirname, extname, resolve } from "node:path"; import { dirname, extname, resolve } from "node:path";
import { fileURLToPath } from "node:url"; import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { import {
assertPhase7iEnvironmentSafe, assertPhase7iEnvironmentSafe,
@@ -10,6 +12,7 @@ import {
phase7iCapabilityCookieForOrigin, phase7iCapabilityCookieForOrigin,
Phase7iDemoRuntime, Phase7iDemoRuntime,
} from "./phase7i-demo-server.js"; } from "./phase7i-demo-server.js";
import { Phase7ChatSession } from "./chat-session.js";
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
const sourceRoot = resolve(packageRoot, "src"); const sourceRoot = resolve(packageRoot, "src");
@@ -105,8 +108,400 @@ describe("Phase 7I deployment boundary", () => {
} }
} }
}); });
it("requires the trusted proxy marker and exact forwarded authority over HTTP", async () => {
await withHttpRuntime(async ({ request }) => {
expect(await request("/api/phase7/health", {
headers: { ...proxyHeaders(), "x-phase7i-proxy": "" },
})).toMatchObject({
status: 403,
body: { error: { code: "trusted_proxy_required" } },
});
expect(await request("/api/phase7/health", {
headers: { ...proxyHeaders(), "x-forwarded-proto": "https,http" },
})).toMatchObject({ status: 403, body: { error: { code: "transport_denied" } } });
expect(await request("/api/phase7/health", {
headers: {
...proxyHeaders(),
"x-forwarded-host": "phase7i.example.ts.net,attacker.example",
},
})).toMatchObject({ status: 403, body: { error: { code: "host_denied" } } });
expect(await request("/api/phase7/health", { headers: proxyHeaders() })).toMatchObject({
status: 200,
body: { status: "ok", controlPlane: "phase7-mock" },
});
});
});
it("requires exact Origin and Fetch Metadata on every mutation", async () => {
await withHttpRuntime(async ({ request }) => {
const body = JSON.stringify({ scenarioId: SCENARIO_ID });
const invalidHeaders = [
{ origin: `${PUBLIC_ORIGIN}/` },
{ "sec-fetch-site": "none" },
{ "sec-fetch-site": "cross-site" },
{ "sec-fetch-mode": "same-origin" },
{ "sec-fetch-dest": "document" },
];
for (const replacement of invalidHeaders) {
const response = await request("/api/phase7/sessions", {
method: "POST",
headers: { ...mutationHeaders(), ...replacement },
body,
});
expect(response.status).toBe(403);
expect(response.body).toMatchObject({ error: { code: expect.stringMatching(/^(?:origin|fetch_metadata)_denied$/) } });
}
const missingFetchSite = mutationHeaders();
delete missingFetchSite["sec-fetch-site"];
expect(await request("/api/phase7/sessions", {
method: "POST",
headers: missingFetchSite,
body,
})).toMatchObject({ status: 403, body: { error: { code: "fetch_metadata_denied" } } });
});
});
it("accepts only JSON objects with the exact route fields", async () => {
await withHttpRuntime(async ({ request }) => {
for (const contentType of ["text/plain", "application/jsonp"]) {
expect(await request("/api/phase7/sessions", {
method: "POST",
headers: { ...mutationHeaders(), "content-type": contentType },
body: JSON.stringify({ scenarioId: SCENARIO_ID }),
})).toMatchObject({ status: 415, body: { error: { code: "json_required" } } });
}
expect(await request("/api/phase7/sessions", {
method: "POST",
headers: mutationHeaders(),
body: JSON.stringify({ scenarioId: SCENARIO_ID, adapter: "paperclip" }),
})).toMatchObject({ status: 422, body: { error: { code: "unexpected_field" } } });
});
});
it("returns the same 404 for missing, foreign, or identity-mismatched session authority", async () => {
await withHttpRuntime(async ({ request }) => {
const created = await createSession(request, "alice@example.com");
const failures = [
mutationHeaders("alice@example.com"),
{ ...mutationHeaders("alice@example.com"), cookie: "__Host-paperclip_phase7i=foreign" },
{ ...mutationHeaders("mallory@example.com"), cookie: created.cookie },
];
for (const headers of failures) {
const response = await request(`/api/phase7/sessions/${created.id}/replay`, {
method: "POST",
headers,
body: "{}",
});
expect(response).toMatchObject({
status: 404,
body: { error: { code: "session_not_found", message: "Session was not found." } },
});
}
});
});
it("isolates two sessions even when the tailnet identity is shared", async () => {
await withHttpRuntime(async ({ request }) => {
const first = await createSession(request, "alice@example.com");
const second = await createSession(request, "alice@example.com");
expect(first.id).not.toBe(second.id);
expect(first.cookie).not.toBe(second.cookie);
expect(await replaySession(request, second.id, first.cookie, "alice@example.com")).toMatchObject({
status: 404,
body: { error: { code: "session_not_found" } },
});
expect(await replaySession(request, first.id, first.cookie, "alice@example.com")).toMatchObject({
status: 200,
body: { sessionId: first.id },
});
expect(await replaySession(request, second.id, second.cookie, "alice@example.com")).toMatchObject({
status: 200,
body: { sessionId: second.id },
});
});
});
it("rotates both session id and capability on reset and invalidates the old pair", async () => {
await withHttpRuntime(async ({ request }) => {
const original = await createSession(request, "alice@example.com");
const reset = await request(`/api/phase7/sessions/${original.id}/reset`, {
method: "POST",
headers: { ...mutationHeaders("alice@example.com"), cookie: original.cookie },
body: "{}",
});
expect(reset.status).toBe(200);
const replacementId = sessionId(reset);
const replacementCookie = responseCookie(reset);
expect(replacementId).not.toBe(original.id);
expect(replacementCookie).not.toBe(original.cookie);
expect(await replaySession(request, original.id, original.cookie, "alice@example.com")).toMatchObject({
status: 404,
body: { error: { code: "session_not_found" } },
});
expect(await replaySession(request, replacementId, original.cookie, "alice@example.com")).toMatchObject({
status: 404,
body: { error: { code: "session_not_found" } },
});
expect(await replaySession(request, replacementId, replacementCookie, "alice@example.com")).toMatchObject({
status: 200,
body: { sessionId: replacementId },
});
});
});
it("denies a concurrent second turn without invoking the chat twice", async () => {
let release!: () => void;
let entered!: () => void;
const gate = new Promise<void>((resolveGate) => {
release = resolveGate;
});
const started = new Promise<void>((resolveStarted) => {
entered = resolveStarted;
});
const originalSend = Phase7ChatSession.prototype.send;
const send = vi.spyOn(Phase7ChatSession.prototype, "send").mockImplementationOnce(
async function (this: Phase7ChatSession, prompt: string) {
entered();
await gate;
return await originalSend.call(this, prompt);
},
);
try {
await withHttpRuntime(async ({ request }) => {
const session = await createSession(request, "alice@example.com");
const firstTurn = request(`/api/phase7/sessions/${session.id}/turn`, {
method: "POST",
headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie },
body: JSON.stringify({ prompt: "First turn" }),
});
await started;
expect(await request(`/api/phase7/sessions/${session.id}/turn`, {
method: "POST",
headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie },
body: JSON.stringify({ prompt: "Second turn" }),
})).toMatchObject({ status: 409, body: { error: { code: "turn_active" } } });
release();
expect(await firstTurn).toMatchObject({ status: 200, body: { sessionId: session.id } });
expect(send).toHaveBeenCalledTimes(1);
});
} finally {
release();
send.mockRestore();
}
});
it("enforces prompt, body, session-capacity, and request-rate bounds", async () => {
await withHttpRuntime(async ({ request }) => {
const session = await createSession(request, "alice@example.com");
expect(await request(`/api/phase7/sessions/${session.id}/turn`, {
method: "POST",
headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie },
body: JSON.stringify({ prompt: "x".repeat(16 * 1024 + 1) }),
})).toMatchObject({ status: 422, body: { error: { code: "prompt_too_large" } } });
expect(await request("/api/phase7/sessions", {
method: "POST",
headers: mutationHeaders("body@example.com"),
body: JSON.stringify({ scenarioId: SCENARIO_ID, padding: "x".repeat(64 * 1024) }),
})).toMatchObject({ status: 413, body: { error: { code: "request_too_large" } } });
for (let index = 1; index < 8; index += 1) {
expect((await createSession(request, `capacity-${index}@example.com`)).id).toHaveLength(32);
}
expect(await request("/api/phase7/sessions", {
method: "POST",
headers: mutationHeaders("capacity-overflow@example.com"),
body: JSON.stringify({ scenarioId: SCENARIO_ID }),
})).toMatchObject({ status: 429, body: { error: { code: "session_capacity" } } });
});
await withHttpRuntime(async ({ request }) => {
for (let index = 0; index < 90; index += 1) {
expect((await request(`/api/phase7/missing-${index}`, {
method: "POST",
headers: mutationHeaders("rate@example.com"),
body: "{}",
})).status).toBe(404);
}
expect(await request("/api/phase7/rate-limit", {
method: "POST",
headers: mutationHeaders("rate@example.com"),
body: "{}",
})).toMatchObject({ status: 429, body: { error: { code: "rate_limited" } } });
});
});
it("returns stable redacted internal errors and security headers", async () => {
await withHttpRuntime(async ({ request }) => {
const first = await request("/%", { headers: proxyHeaders() });
const second = await request("/%E0%A4%A", { headers: proxyHeaders() });
expect(first).toMatchObject({
status: 500,
body: { error: { code: "internal_error", message: "The request could not be completed." } },
});
expect(second.body).toEqual(first.body);
expect(JSON.stringify(first.body)).not.toMatch(/URIError|decodeURIComponent|%E0|stack/i);
for (const response of [first, await request("/api/phase7/health", { headers: proxyHeaders() })]) {
expect(response.headers.get("cache-control")).toBe("no-store");
expect(response.headers.get("content-security-policy")).toContain("default-src 'none'");
expect(response.headers.get("cross-origin-opener-policy")).toBe("same-origin");
expect(response.headers.get("cross-origin-resource-policy")).toBe("same-origin");
expect(response.headers.get("permissions-policy")).toContain("camera=()");
expect(response.headers.get("referrer-policy")).toBe("no-referrer");
expect(response.headers.get("strict-transport-security")).toBe("max-age=31536000");
expect(response.headers.get("x-content-type-options")).toBe("nosniff");
expect(response.headers.get("x-frame-options")).toBe("DENY");
}
});
});
}); });
const PUBLIC_ORIGIN = "https://phase7i.example.ts.net";
const SCENARIO_ID = "hb-inbox-lite-01";
type HttpResult = {
status: number;
headers: Headers;
body: unknown;
};
type HttpRequest = (
path: string,
init?: { method?: string; headers?: Record<string, string>; body?: string },
) => Promise<HttpResult>;
function proxyHeaders(identity = "alice@example.com"): Record<string, string> {
return {
"tailscale-user-login": identity,
"x-forwarded-host": new URL(PUBLIC_ORIGIN).host,
"x-forwarded-proto": new URL(PUBLIC_ORIGIN).protocol.slice(0, -1),
"x-phase7i-proxy": "v1",
};
}
function mutationHeaders(identity = "alice@example.com"): Record<string, string> {
return {
...proxyHeaders(identity),
"content-type": "application/json; charset=utf-8",
origin: PUBLIC_ORIGIN,
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
};
}
async function withHttpRuntime(
run: (context: { runtime: Phase7iDemoRuntime; request: HttpRequest }) => Promise<void>,
): Promise<void> {
const runtime = new Phase7iDemoRuntime({
publicOrigin: PUBLIC_ORIGIN,
basePath: "",
commitSha: "deadbeef",
packageRoot,
log: () => undefined,
});
await runtime.initialize();
const server = runtime.createServer();
const scratchRoot = process.env.PAPERCLIP_RUN_SCRATCH_DIR ?? tmpdir();
const socketDirectory = await mkdtemp(resolve(scratchRoot, "phase7i-http-"));
const socketPath = resolve(socketDirectory, "server.sock");
await new Promise<void>((resolveListen, rejectListen) => {
server.once("error", rejectListen);
server.listen(socketPath, () => {
server.off("error", rejectListen);
resolveListen();
});
});
const request: HttpRequest = async (path, init = {}) => {
return await new Promise<HttpResult>((resolveRequest, rejectRequest) => {
const request = httpRequest({
socketPath,
path,
method: init.method ?? "GET",
headers: init.headers,
}, (response) => {
const chunks: Buffer[] = [];
response.on("data", (chunk: Buffer | string) => {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
});
response.once("error", rejectRequest);
response.once("end", () => {
const headers = new Headers();
for (const [name, value] of Object.entries(response.headers)) {
if (Array.isArray(value)) {
for (const item of value) headers.append(name, item);
} else if (value !== undefined) {
headers.set(name, String(value));
}
}
const text = Buffer.concat(chunks).toString("utf8");
resolveRequest({
status: response.statusCode ?? 0,
headers,
body: text === "" ? null : JSON.parse(text) as unknown,
});
});
});
request.once("error", rejectRequest);
if (init.body !== undefined) request.write(init.body);
request.end();
});
};
try {
await run({ runtime, request });
} finally {
await new Promise<void>((resolveClose, rejectClose) => {
server.close((error) => error ? rejectClose(error) : resolveClose());
});
await runtime.shutdown();
await rm(socketDirectory, { recursive: true, force: true });
}
}
async function createSession(request: HttpRequest, identity: string): Promise<{ id: string; cookie: string }> {
const response = await request("/api/phase7/sessions", {
method: "POST",
headers: mutationHeaders(identity),
body: JSON.stringify({ scenarioId: SCENARIO_ID }),
});
expect(response.status).toBe(201);
return { id: sessionId(response), cookie: responseCookie(response) };
}
function sessionId(response: HttpResult): string {
if (
typeof response.body !== "object" ||
response.body === null ||
!("sessionId" in response.body) ||
typeof response.body.sessionId !== "string"
) {
throw new Error("response did not contain a session id");
}
return response.body.sessionId;
}
function responseCookie(response: HttpResult): string {
const setCookie = response.headers.get("set-cookie");
if (!setCookie) throw new Error("response did not set a capability cookie");
return setCookie.split(";", 1)[0]!;
}
async function replaySession(
request: HttpRequest,
id: string,
cookie: string,
identity: string,
): Promise<HttpResult> {
return await request(`/api/phase7/sessions/${id}/replay`, {
method: "POST",
headers: { ...mutationHeaders(identity), cookie },
body: "{}",
});
}
function importSpecifiers(source: string): string[] { function importSpecifiers(source: string): string[] {
const matches = source.matchAll(/\b(?:import|export)\s+(?:type\s+)?(?:[^"'`;]{0,500}?\s+from\s+)?["']([^"']+)["']/g); const matches = source.matchAll(/\b(?:import|export)\s+(?:type\s+)?(?:[^"'`;]{0,500}?\s+from\s+)?["']([^"']+)["']/g);
return [...matches].map((match) => match[1]!); return [...matches].map((match) => match[1]!);
@@ -159,7 +159,8 @@ function sendError(res: ServerResponse, error: RequestFailure): void {
} }
async function readJsonBody(req: IncomingMessage): Promise<Record<string, unknown>> { async function readJsonBody(req: IncomingMessage): Promise<Record<string, unknown>> {
if (!header(req, "content-type").toLowerCase().startsWith("application/json")) { const mediaType = header(req, "content-type").split(";", 1)[0]?.trim().toLowerCase();
if (mediaType !== "application/json") {
throw new RequestFailure(415, "json_required", "Mutation requests require application/json."); throw new RequestFailure(415, "json_required", "Mutation requests require application/json.");
} }
const declaredLength = Number(header(req, "content-length")); const declaredLength = Number(header(req, "content-length"));
@@ -359,11 +360,11 @@ export class Phase7iDemoRuntime {
if (header(req, "x-phase7i-proxy") !== "v1") { if (header(req, "x-phase7i-proxy") !== "v1") {
throw new RequestFailure(403, "trusted_proxy_required", "Request did not arrive through the trusted proxy."); throw new RequestFailure(403, "trusted_proxy_required", "Request did not arrive through the trusted proxy.");
} }
const forwardedProtocol = header(req, "x-forwarded-proto").split(",")[0]?.trim().toLowerCase(); const forwardedProtocol = header(req, "x-forwarded-proto").toLowerCase();
if (forwardedProtocol !== this.publicOrigin.protocol.slice(0, -1)) { if (forwardedProtocol !== this.publicOrigin.protocol.slice(0, -1)) {
throw new RequestFailure(403, "transport_denied", "Forwarded transport is not allowed."); throw new RequestFailure(403, "transport_denied", "Forwarded transport is not allowed.");
} }
const forwardedHost = header(req, "x-forwarded-host").split(",")[0]?.trim().toLowerCase(); const forwardedHost = header(req, "x-forwarded-host").toLowerCase();
if (forwardedHost !== this.publicOrigin.host.toLowerCase()) { if (forwardedHost !== this.publicOrigin.host.toLowerCase()) {
throw new RequestFailure(403, "host_denied", "Forwarded host is not allowed."); throw new RequestFailure(403, "host_denied", "Forwarded host is not allowed.");
} }
@@ -374,9 +375,12 @@ export class Phase7iDemoRuntime {
if (header(req, "origin") !== this.publicOrigin.origin) { if (header(req, "origin") !== this.publicOrigin.origin) {
throw new RequestFailure(403, "origin_denied", "Request origin is not allowed."); throw new RequestFailure(403, "origin_denied", "Request origin is not allowed.");
} }
const fetchSite = header(req, "sec-fetch-site").toLowerCase(); if (
if (fetchSite && fetchSite !== "same-origin" && fetchSite !== "none") { header(req, "sec-fetch-site").toLowerCase() !== "same-origin" ||
throw new RequestFailure(403, "fetch_metadata_denied", "Cross-site mutation requests are not allowed."); header(req, "sec-fetch-mode").toLowerCase() !== "cors" ||
header(req, "sec-fetch-dest").toLowerCase() !== "empty"
) {
throw new RequestFailure(403, "fetch_metadata_denied", "Mutation request metadata is not allowed.");
} }
} }
@@ -257,17 +257,6 @@ describe("Phase 7 exposure and authorization", () => {
}); });
expect(revisions).toMatchObject({ ok: true, value: [{ id: revisionId, revision: 1 }] }); expect(revisions).toMatchObject({ ok: true, value: [{ id: revisionId, revision: 1 }] });
expect(await runtime.invoke({
operationId: "request_human_input",
input: {
interactionKind: "confirmation",
title: "Confirm plan",
prompt: "Accept this plan?",
targetRevisionId: revisionId!,
continuationPolicy: "wake_assignee",
},
idempotencyKey: "confirm-plan-1",
})).toMatchObject({ ok: true });
expect(await runtime.invoke({ expect(await runtime.invoke({
operationId: "register_deliverable", operationId: "register_deliverable",
input: { input: {
@@ -285,6 +274,17 @@ describe("Phase 7 exposure and authorization", () => {
input: { summary: "Ready for semantic review." }, input: { summary: "Ready for semantic review." },
idempotencyKey: "review-1", idempotencyKey: "review-1",
})).toMatchObject({ ok: true }); })).toMatchObject({ ok: true });
expect(await runtime.invoke({
operationId: "request_human_input",
input: {
interactionKind: "confirmation",
title: "Confirm plan",
prompt: "Accept this plan?",
targetRevisionId: revisionId!,
continuationPolicy: "wake_assignee",
},
idempotencyKey: "confirm-plan-1",
})).toMatchObject({ ok: true });
expect(adapter.snapshot()).toMatchObject({ expect(adapter.snapshot()).toMatchObject({
tasks: [{ status: "in_review" }], tasks: [{ status: "in_review" }],
@@ -294,6 +294,34 @@ describe("Phase 7 exposure and authorization", () => {
workProducts: [{ type: "artifact" }], workProducts: [{ type: "artifact" }],
}); });
}); });
it("keeps a pending human-input handoff in review instead of weakening the lifecycle", async () => {
const { adapter, runtime } = await runtimeFor();
expect(await runtime.invoke({
operationId: "request_human_input",
input: {
interactionKind: "confirmation",
title: "Confirm plan",
prompt: "Accept this plan?",
continuationPolicy: "wake_assignee",
},
idempotencyKey: "confirm-plan-first",
})).toMatchObject({ ok: true });
expect(await runtime.invoke({
operationId: "request_review",
input: { summary: "A duplicate review transition must fail closed." },
idempotencyKey: "duplicate-review",
})).toMatchObject({
ok: false,
error: { code: "operation_unsupported", reason: "mock_operation_rejected" },
});
expect(adapter.snapshot()).toMatchObject({
tasks: [{ status: "in_review" }],
interactions: [{ status: "pending" }],
comments: [],
});
});
}); });
describe("Phase 7 security policy", () => { describe("Phase 7 security policy", () => {