mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 02:44:50 +01:00
fix(runner): close Phase 7 security gate
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
fe231fbb7e
commit
865d1a7234
8 files changed
+597
-25
No files matched your search
@@ -69,6 +69,14 @@ pnpm --filter @paperclipai/paperclip-runner report:phase7-evals
|
||||
```
|
||||
|
||||
The reporter writes `knowledge/evidence/phase-07/eval-parity-report.{json,md}`.
|
||||
|
||||
Run the bounded provider conformance matrix separately. It creates exactly one
|
||||
real Codex turn for each of the 16 checked-in eval groups while retaining the
|
||||
in-process mock control plane:
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/paperclip-runner report:phase7-live-evals
|
||||
```
|
||||
Each failure carries its case ID, assertion class, semantic operation,
|
||||
authorization decision, and final state diff. The report is generated on demand
|
||||
and is not committed; delete it before running `docs:validate` (it carries no
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"test:phase7-inventory": "node scripts/check-phase7-inventory.test.mjs",
|
||||
"test:phase7-evals": "vitest run src/conformance/phase7-eval-suite.test.ts",
|
||||
"report:phase7-evals": "pnpm run build:typescript && node scripts/run-phase7-eval-suite.mjs",
|
||||
"report:phase7-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-phase7-live-eval-matrix.mjs",
|
||||
"check:phase0-parity": "vitest run src/tracer/phase0-runner.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output && node -e \"process.stdout.write('Rust and TypeScript Phase 0 tracer output matches the shared golden fixture.\\n')\"",
|
||||
"check:phase1-parity": "vitest run src/reducer/session-reducer.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core phase1_fixture_parity && node -e \"process.stdout.write('Rust and TypeScript Phase 1 reducer summaries match the shared golden fixtures.\\n')\"",
|
||||
"docs:validate": "node scripts/validate-doc-links.mjs && node scripts/validate-okf.mjs",
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
const packageRoot = resolve(import.meta.dirname, "..");
|
||||
const suite = await import(resolve(packageRoot, "dist/conformance/phase7-eval-suite.js"));
|
||||
const matrix = await suite.runPhase7LiveCodexMatrix(process.cwd());
|
||||
const report = {
|
||||
schema: "paperclip.phase7.live-codex-matrix.v1",
|
||||
groups: matrix.length,
|
||||
matrix,
|
||||
};
|
||||
const output = resolve(packageRoot, "knowledge/evidence/phase-07/live-codex-matrix.json");
|
||||
const markdown = output.replace(/\.json$/, ".md");
|
||||
await mkdir(dirname(output), { recursive: true });
|
||||
await Promise.all([
|
||||
writeFile(output, `${JSON.stringify(report, null, 2)}\n`),
|
||||
writeFile(markdown, [
|
||||
"# Phase 7 Live Codex Matrix",
|
||||
"",
|
||||
`- Groups: ${report.groups}`,
|
||||
`- Cases: ${matrix.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`,
|
||||
"- Each row asserts expected and observed typed calls plus final mock state.",
|
||||
"",
|
||||
].join("\n")),
|
||||
]);
|
||||
process.stdout.write(`Phase 7 live Codex matrix passed: ${matrix.length} groups.\n`);
|
||||
process.stdout.write(`Live matrix reports: ${output} and ${markdown}\n`);
|
||||
@@ -15,7 +15,13 @@ describe("Phase 7 offline eval conformance", () => {
|
||||
"restraint_no_call",
|
||||
]);
|
||||
expect(report.fakeAgentOperationCount).toBe(report.codexOperationCount);
|
||||
expect(report.boundedCodexSample).toHaveLength(9);
|
||||
expect(report.results.every((result) => result.caseId && result.semanticOperation && result.authorizationDecision)).toBe(true);
|
||||
expect(report.boundedCodexSample).toHaveLength(16);
|
||||
expect(report.results.every((result) =>
|
||||
result.caseId &&
|
||||
result.semanticOperation &&
|
||||
result.authorizationDecision &&
|
||||
result.expectedSemantics.length > 0 &&
|
||||
result.finalState.expected === result.finalState.observed,
|
||||
)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,7 @@ import type { Phase7FixtureSeed, Phase7JsonValue } from "../mock-core/phase7-con
|
||||
import { Phase7SemanticToolRuntime } from "../tools/phase7-semantic-tool-runtime.js";
|
||||
import { Phase7CodexToolBinding, Phase7FakeAgentToolBinding } from "../tools/phase7-tool-bindings.js";
|
||||
import { phase7FixtureRunCapabilities } from "../phase7/fixture-run-capabilities.js";
|
||||
import { Phase7LiveSessionService } from "../phase7/live-session.js";
|
||||
|
||||
const GENERATED_HEADER = "# GENERATED FILE — DO NOT EDIT. Run pnpm generate:phase7-inventory.\n";
|
||||
const PACKAGE_ROOT = resolve(import.meta.dirname, "../..");
|
||||
@@ -41,11 +42,24 @@ export interface Phase7EvalCaseResult {
|
||||
group: string;
|
||||
assertionClasses: AssertionClass[];
|
||||
semanticOperation: string;
|
||||
expectedSemantics: string[];
|
||||
forbiddenSemantics: string[];
|
||||
authorizationDecision: string;
|
||||
stateDiff: string[];
|
||||
finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" };
|
||||
sourceAnchor: string;
|
||||
}
|
||||
|
||||
export interface Phase7LiveCodexMatrixResult {
|
||||
caseId: string;
|
||||
group: string;
|
||||
semanticOperation: string;
|
||||
expectedCalls: string[];
|
||||
forbiddenCalls: string[];
|
||||
observedCalls: string[];
|
||||
finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" };
|
||||
}
|
||||
|
||||
export interface Phase7EvalParityReport {
|
||||
schema: "paperclip.phase7.eval-parity-report.v1";
|
||||
cases: number;
|
||||
@@ -161,6 +175,7 @@ async function runCase(row: Phase7EvalRow): Promise<Phase7EvalCaseResult> {
|
||||
}
|
||||
|
||||
const stateDiff = before === after ? [] : ["mock_state.revision"];
|
||||
const expectedState = operationMutatesState(semanticOperation) ? "mutated" : "unchanged";
|
||||
if (row.assertionClasses.includes("restraint_no_call") && stateDiff.length !== 0) {
|
||||
throw failure(row, semanticOperation, authorizationDecision, stateDiff);
|
||||
}
|
||||
@@ -170,8 +185,14 @@ async function runCase(row: Phase7EvalRow): Promise<Phase7EvalCaseResult> {
|
||||
group: row.group,
|
||||
assertionClasses: row.assertionClasses,
|
||||
semanticOperation,
|
||||
expectedSemantics: row.expectedSemantics,
|
||||
forbiddenSemantics: row.forbiddenSemantics,
|
||||
authorizationDecision,
|
||||
stateDiff,
|
||||
finalState: {
|
||||
expected: expectedState,
|
||||
observed: stateDiff.length === 0 ? "unchanged" : "mutated",
|
||||
},
|
||||
sourceAnchor: row.sourceAnchor,
|
||||
};
|
||||
}
|
||||
@@ -191,7 +212,7 @@ function operationPlan(row: Phase7EvalRow): OperationPlan {
|
||||
bl: { operationId: "block_task", input: { reason: marker } },
|
||||
dp: { operationId: "write_document", input: { key: `plan-${row.id}`, title: row.title, body: marker, baseRevisionId: null } },
|
||||
ix: { operationId: "request_human_input", input: { interactionKind: "confirmation", title: row.title, prompt: marker, continuationPolicy: "wake_assignee" } },
|
||||
ar: { operationId: "register_deliverable", input: { filename: `${row.id}.json`, contentType: "application/json", byteSize: 2, sha256: "00", contentRef: `memory://${row.id}`, title: row.title } },
|
||||
ar: { operationId: "register_deliverable", input: { filename: `${row.id}.json`, contentType: "application/json", byteSize: 2, sha256: "0".repeat(64), contentRef: `memory://${row.id}`, title: row.title } },
|
||||
};
|
||||
const always = alwaysByGroup[row.group];
|
||||
if (always) return { ...always, grants: [] };
|
||||
@@ -240,7 +261,7 @@ async function runtimeFor(actorGrants: string[] = []) {
|
||||
}
|
||||
|
||||
function selectCodexSample(results: Phase7EvalCaseResult[]): Array<{ caseId: string; group: string; semanticOperation: string }> {
|
||||
const groups = ["hb", "dp", "bl", "ap", "ar", "ix", "mh", "rs", "wk"];
|
||||
const groups = [...new Set(results.map((result) => result.group))].sort();
|
||||
return groups.map((group) => {
|
||||
const result = results.find((candidate) => candidate.group === group);
|
||||
if (!result) throw new Error(`missing bounded Codex sample group ${group}`);
|
||||
@@ -248,6 +269,88 @@ function selectCodexSample(results: Phase7EvalCaseResult[]): Array<{ caseId: str
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs one bounded, real-Codex turn for each checked-in eval group. */
|
||||
export async function runPhase7LiveCodexMatrix(
|
||||
workingDirectory = process.cwd(),
|
||||
): Promise<Phase7LiveCodexMatrixResult[]> {
|
||||
const inventory = await readInventory();
|
||||
assertInventory(inventory);
|
||||
const matrix: Phase7LiveCodexMatrixResult[] = [];
|
||||
|
||||
for (const group of [...new Set(inventory.rows.map((row) => row.group))].sort()) {
|
||||
const row = inventory.rows.find((candidate) => candidate.group === group);
|
||||
if (!row) throw new Error(`missing live Codex representative for group ${group}`);
|
||||
const plan = operationPlan(row);
|
||||
const service = new Phase7LiveSessionService();
|
||||
const session = await service.create({
|
||||
workingDirectory,
|
||||
seed: liveFixtureSeed(plan.grants),
|
||||
capabilities: phase7FixtureRunCapabilities(plan.grants),
|
||||
explicitClaims: plan.grants,
|
||||
turnTimeoutMs: 60_000,
|
||||
scenario: { id: `phase7-live-eval-${row.id}` },
|
||||
});
|
||||
try {
|
||||
const initialMockState = session.snapshot().mockState;
|
||||
const expectedCalls = row.primaryDisposition === "control_plane_owned" ? [] : [plan.operationId];
|
||||
const liveInput = {
|
||||
...(typeof plan.input === "object" && plan.input !== null && !Array.isArray(plan.input) ? plan.input : {}),
|
||||
idempotencyKey: `phase7-live:${row.id}`,
|
||||
};
|
||||
const prompt = expectedCalls.length === 0
|
||||
? "Do not call any tools. Reply with exactly: control-plane-owned."
|
||||
: `Call ${plan.operationId} exactly once with this JSON input: ${JSON.stringify(liveInput)}. Do not call any other tool. Reply after the typed result.`;
|
||||
await session.sendMessage(prompt);
|
||||
const snapshot = session.snapshot();
|
||||
const observedCalls = snapshot.evidence
|
||||
.filter((entry) => entry.kind === "tool_call")
|
||||
.map((entry) => String(entry.data.operationId));
|
||||
const expectedState = operationMutatesState(plan.operationId) ? "mutated" : "unchanged";
|
||||
const observedState = snapshot.mockState === initialMockState ? "unchanged" : "mutated";
|
||||
if (JSON.stringify(observedCalls) !== JSON.stringify(expectedCalls) || observedState !== expectedState) {
|
||||
throw new Error(JSON.stringify({
|
||||
caseId: row.id,
|
||||
expectedCalls,
|
||||
observedCalls,
|
||||
expectedState,
|
||||
observedState,
|
||||
toolResults: snapshot.evidence.filter((entry) => entry.kind === "tool_result").map((entry) => entry.data.result),
|
||||
}));
|
||||
}
|
||||
matrix.push({
|
||||
caseId: row.id,
|
||||
group,
|
||||
semanticOperation: plan.operationId,
|
||||
expectedCalls,
|
||||
forbiddenCalls: row.forbiddenSemantics,
|
||||
observedCalls,
|
||||
finalState: { expected: expectedState, observed: observedState },
|
||||
});
|
||||
} finally {
|
||||
await service.shutdown(session.id, `Phase 7 live eval ${row.id} complete`);
|
||||
}
|
||||
}
|
||||
return matrix;
|
||||
}
|
||||
|
||||
function liveFixtureSeed(actorGrants: string[]): Phase7FixtureSeed {
|
||||
return {
|
||||
actors: [{
|
||||
id: "actor-1",
|
||||
companyId: "company-1",
|
||||
name: "Eval fixture actor",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
budgetId: "budget-actor-1",
|
||||
capabilityGrants: actorGrants,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
function operationMutatesState(operationId: string): boolean {
|
||||
return !["checkout_task", "search_tasks", "list_agents"].includes(operationId);
|
||||
}
|
||||
|
||||
async function toolSurfaceParity(sample: Array<{ caseId: string; group: string; semanticOperation: string }>) {
|
||||
const { runtime } = await runtimeFor([
|
||||
"discovery:tasks:read",
|
||||
@@ -290,13 +393,13 @@ function renderMarkdown(report: Phase7EvalParityReport): string {
|
||||
`- Groups: ${report.groups.length} (${report.groups.join(", ")})`,
|
||||
`- Assertion classes: ${report.assertionClasses.join(", ")}`,
|
||||
`- Fake-agent/Codex operations: ${report.fakeAgentOperationCount}/${report.codexOperationCount}`,
|
||||
`- Bounded Codex binding sample: ${report.boundedCodexSample.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`,
|
||||
`- Bounded Codex binding matrix: ${report.boundedCodexSample.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`,
|
||||
`- Semantic execution: ${Object.entries(byDisposition).map(([operation, count]) => `${operation}=${count}`).join(", ")}`,
|
||||
"",
|
||||
"## Offline Guarantees",
|
||||
"",
|
||||
"- The suite reads only the checked-in Phase 7 traceability derivative and starts only the in-process mock adapter.",
|
||||
"- Each failure carries case ID, assertion class, semantic operation, authorization decision, and final state diff.",
|
||||
"- Each result includes source expected/forbidden semantics plus expected and observed final mock state.",
|
||||
"- Optional operations are evaluated in both granted and absent configurations; control-plane-owned operations remain absent.",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { request as httpRequest } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, extname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
assertPhase7iEnvironmentSafe,
|
||||
@@ -10,6 +12,7 @@ import {
|
||||
phase7iCapabilityCookieForOrigin,
|
||||
Phase7iDemoRuntime,
|
||||
} from "./phase7i-demo-server.js";
|
||||
import { Phase7ChatSession } from "./chat-session.js";
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const sourceRoot = resolve(packageRoot, "src");
|
||||
@@ -105,8 +108,400 @@ describe("Phase 7I deployment boundary", () => {
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("requires the trusted proxy marker and exact forwarded authority over HTTP", async () => {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
expect(await request("/api/phase7/health", {
|
||||
headers: { ...proxyHeaders(), "x-phase7i-proxy": "" },
|
||||
})).toMatchObject({
|
||||
status: 403,
|
||||
body: { error: { code: "trusted_proxy_required" } },
|
||||
});
|
||||
expect(await request("/api/phase7/health", {
|
||||
headers: { ...proxyHeaders(), "x-forwarded-proto": "https,http" },
|
||||
})).toMatchObject({ status: 403, body: { error: { code: "transport_denied" } } });
|
||||
expect(await request("/api/phase7/health", {
|
||||
headers: {
|
||||
...proxyHeaders(),
|
||||
"x-forwarded-host": "phase7i.example.ts.net,attacker.example",
|
||||
},
|
||||
})).toMatchObject({ status: 403, body: { error: { code: "host_denied" } } });
|
||||
expect(await request("/api/phase7/health", { headers: proxyHeaders() })).toMatchObject({
|
||||
status: 200,
|
||||
body: { status: "ok", controlPlane: "phase7-mock" },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("requires exact Origin and Fetch Metadata on every mutation", async () => {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
const body = JSON.stringify({ scenarioId: SCENARIO_ID });
|
||||
const invalidHeaders = [
|
||||
{ origin: `${PUBLIC_ORIGIN}/` },
|
||||
{ "sec-fetch-site": "none" },
|
||||
{ "sec-fetch-site": "cross-site" },
|
||||
{ "sec-fetch-mode": "same-origin" },
|
||||
{ "sec-fetch-dest": "document" },
|
||||
];
|
||||
for (const replacement of invalidHeaders) {
|
||||
const response = await request("/api/phase7/sessions", {
|
||||
method: "POST",
|
||||
headers: { ...mutationHeaders(), ...replacement },
|
||||
body,
|
||||
});
|
||||
expect(response.status).toBe(403);
|
||||
expect(response.body).toMatchObject({ error: { code: expect.stringMatching(/^(?:origin|fetch_metadata)_denied$/) } });
|
||||
}
|
||||
const missingFetchSite = mutationHeaders();
|
||||
delete missingFetchSite["sec-fetch-site"];
|
||||
expect(await request("/api/phase7/sessions", {
|
||||
method: "POST",
|
||||
headers: missingFetchSite,
|
||||
body,
|
||||
})).toMatchObject({ status: 403, body: { error: { code: "fetch_metadata_denied" } } });
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts only JSON objects with the exact route fields", async () => {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
for (const contentType of ["text/plain", "application/jsonp"]) {
|
||||
expect(await request("/api/phase7/sessions", {
|
||||
method: "POST",
|
||||
headers: { ...mutationHeaders(), "content-type": contentType },
|
||||
body: JSON.stringify({ scenarioId: SCENARIO_ID }),
|
||||
})).toMatchObject({ status: 415, body: { error: { code: "json_required" } } });
|
||||
}
|
||||
expect(await request("/api/phase7/sessions", {
|
||||
method: "POST",
|
||||
headers: mutationHeaders(),
|
||||
body: JSON.stringify({ scenarioId: SCENARIO_ID, adapter: "paperclip" }),
|
||||
})).toMatchObject({ status: 422, body: { error: { code: "unexpected_field" } } });
|
||||
});
|
||||
});
|
||||
|
||||
it("returns the same 404 for missing, foreign, or identity-mismatched session authority", async () => {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
const created = await createSession(request, "alice@example.com");
|
||||
const failures = [
|
||||
mutationHeaders("alice@example.com"),
|
||||
{ ...mutationHeaders("alice@example.com"), cookie: "__Host-paperclip_phase7i=foreign" },
|
||||
{ ...mutationHeaders("mallory@example.com"), cookie: created.cookie },
|
||||
];
|
||||
for (const headers of failures) {
|
||||
const response = await request(`/api/phase7/sessions/${created.id}/replay`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: "{}",
|
||||
});
|
||||
expect(response).toMatchObject({
|
||||
status: 404,
|
||||
body: { error: { code: "session_not_found", message: "Session was not found." } },
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("isolates two sessions even when the tailnet identity is shared", async () => {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
const first = await createSession(request, "alice@example.com");
|
||||
const second = await createSession(request, "alice@example.com");
|
||||
expect(first.id).not.toBe(second.id);
|
||||
expect(first.cookie).not.toBe(second.cookie);
|
||||
|
||||
expect(await replaySession(request, second.id, first.cookie, "alice@example.com")).toMatchObject({
|
||||
status: 404,
|
||||
body: { error: { code: "session_not_found" } },
|
||||
});
|
||||
expect(await replaySession(request, first.id, first.cookie, "alice@example.com")).toMatchObject({
|
||||
status: 200,
|
||||
body: { sessionId: first.id },
|
||||
});
|
||||
expect(await replaySession(request, second.id, second.cookie, "alice@example.com")).toMatchObject({
|
||||
status: 200,
|
||||
body: { sessionId: second.id },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("rotates both session id and capability on reset and invalidates the old pair", async () => {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
const original = await createSession(request, "alice@example.com");
|
||||
const reset = await request(`/api/phase7/sessions/${original.id}/reset`, {
|
||||
method: "POST",
|
||||
headers: { ...mutationHeaders("alice@example.com"), cookie: original.cookie },
|
||||
body: "{}",
|
||||
});
|
||||
expect(reset.status).toBe(200);
|
||||
const replacementId = sessionId(reset);
|
||||
const replacementCookie = responseCookie(reset);
|
||||
expect(replacementId).not.toBe(original.id);
|
||||
expect(replacementCookie).not.toBe(original.cookie);
|
||||
|
||||
expect(await replaySession(request, original.id, original.cookie, "alice@example.com")).toMatchObject({
|
||||
status: 404,
|
||||
body: { error: { code: "session_not_found" } },
|
||||
});
|
||||
expect(await replaySession(request, replacementId, original.cookie, "alice@example.com")).toMatchObject({
|
||||
status: 404,
|
||||
body: { error: { code: "session_not_found" } },
|
||||
});
|
||||
expect(await replaySession(request, replacementId, replacementCookie, "alice@example.com")).toMatchObject({
|
||||
status: 200,
|
||||
body: { sessionId: replacementId },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("denies a concurrent second turn without invoking the chat twice", async () => {
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const gate = new Promise<void>((resolveGate) => {
|
||||
release = resolveGate;
|
||||
});
|
||||
const started = new Promise<void>((resolveStarted) => {
|
||||
entered = resolveStarted;
|
||||
});
|
||||
const originalSend = Phase7ChatSession.prototype.send;
|
||||
const send = vi.spyOn(Phase7ChatSession.prototype, "send").mockImplementationOnce(
|
||||
async function (this: Phase7ChatSession, prompt: string) {
|
||||
entered();
|
||||
await gate;
|
||||
return await originalSend.call(this, prompt);
|
||||
},
|
||||
);
|
||||
try {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
const session = await createSession(request, "alice@example.com");
|
||||
const firstTurn = request(`/api/phase7/sessions/${session.id}/turn`, {
|
||||
method: "POST",
|
||||
headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie },
|
||||
body: JSON.stringify({ prompt: "First turn" }),
|
||||
});
|
||||
await started;
|
||||
expect(await request(`/api/phase7/sessions/${session.id}/turn`, {
|
||||
method: "POST",
|
||||
headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie },
|
||||
body: JSON.stringify({ prompt: "Second turn" }),
|
||||
})).toMatchObject({ status: 409, body: { error: { code: "turn_active" } } });
|
||||
release();
|
||||
expect(await firstTurn).toMatchObject({ status: 200, body: { sessionId: session.id } });
|
||||
expect(send).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
} finally {
|
||||
release();
|
||||
send.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("enforces prompt, body, session-capacity, and request-rate bounds", async () => {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
const session = await createSession(request, "alice@example.com");
|
||||
expect(await request(`/api/phase7/sessions/${session.id}/turn`, {
|
||||
method: "POST",
|
||||
headers: { ...mutationHeaders("alice@example.com"), cookie: session.cookie },
|
||||
body: JSON.stringify({ prompt: "x".repeat(16 * 1024 + 1) }),
|
||||
})).toMatchObject({ status: 422, body: { error: { code: "prompt_too_large" } } });
|
||||
expect(await request("/api/phase7/sessions", {
|
||||
method: "POST",
|
||||
headers: mutationHeaders("body@example.com"),
|
||||
body: JSON.stringify({ scenarioId: SCENARIO_ID, padding: "x".repeat(64 * 1024) }),
|
||||
})).toMatchObject({ status: 413, body: { error: { code: "request_too_large" } } });
|
||||
|
||||
for (let index = 1; index < 8; index += 1) {
|
||||
expect((await createSession(request, `capacity-${index}@example.com`)).id).toHaveLength(32);
|
||||
}
|
||||
expect(await request("/api/phase7/sessions", {
|
||||
method: "POST",
|
||||
headers: mutationHeaders("capacity-overflow@example.com"),
|
||||
body: JSON.stringify({ scenarioId: SCENARIO_ID }),
|
||||
})).toMatchObject({ status: 429, body: { error: { code: "session_capacity" } } });
|
||||
});
|
||||
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
for (let index = 0; index < 90; index += 1) {
|
||||
expect((await request(`/api/phase7/missing-${index}`, {
|
||||
method: "POST",
|
||||
headers: mutationHeaders("rate@example.com"),
|
||||
body: "{}",
|
||||
})).status).toBe(404);
|
||||
}
|
||||
expect(await request("/api/phase7/rate-limit", {
|
||||
method: "POST",
|
||||
headers: mutationHeaders("rate@example.com"),
|
||||
body: "{}",
|
||||
})).toMatchObject({ status: 429, body: { error: { code: "rate_limited" } } });
|
||||
});
|
||||
});
|
||||
|
||||
it("returns stable redacted internal errors and security headers", async () => {
|
||||
await withHttpRuntime(async ({ request }) => {
|
||||
const first = await request("/%", { headers: proxyHeaders() });
|
||||
const second = await request("/%E0%A4%A", { headers: proxyHeaders() });
|
||||
expect(first).toMatchObject({
|
||||
status: 500,
|
||||
body: { error: { code: "internal_error", message: "The request could not be completed." } },
|
||||
});
|
||||
expect(second.body).toEqual(first.body);
|
||||
expect(JSON.stringify(first.body)).not.toMatch(/URIError|decodeURIComponent|%E0|stack/i);
|
||||
|
||||
for (const response of [first, await request("/api/phase7/health", { headers: proxyHeaders() })]) {
|
||||
expect(response.headers.get("cache-control")).toBe("no-store");
|
||||
expect(response.headers.get("content-security-policy")).toContain("default-src 'none'");
|
||||
expect(response.headers.get("cross-origin-opener-policy")).toBe("same-origin");
|
||||
expect(response.headers.get("cross-origin-resource-policy")).toBe("same-origin");
|
||||
expect(response.headers.get("permissions-policy")).toContain("camera=()");
|
||||
expect(response.headers.get("referrer-policy")).toBe("no-referrer");
|
||||
expect(response.headers.get("strict-transport-security")).toBe("max-age=31536000");
|
||||
expect(response.headers.get("x-content-type-options")).toBe("nosniff");
|
||||
expect(response.headers.get("x-frame-options")).toBe("DENY");
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const PUBLIC_ORIGIN = "https://phase7i.example.ts.net";
|
||||
const SCENARIO_ID = "hb-inbox-lite-01";
|
||||
|
||||
type HttpResult = {
|
||||
status: number;
|
||||
headers: Headers;
|
||||
body: unknown;
|
||||
};
|
||||
|
||||
type HttpRequest = (
|
||||
path: string,
|
||||
init?: { method?: string; headers?: Record<string, string>; body?: string },
|
||||
) => Promise<HttpResult>;
|
||||
|
||||
function proxyHeaders(identity = "alice@example.com"): Record<string, string> {
|
||||
return {
|
||||
"tailscale-user-login": identity,
|
||||
"x-forwarded-host": new URL(PUBLIC_ORIGIN).host,
|
||||
"x-forwarded-proto": new URL(PUBLIC_ORIGIN).protocol.slice(0, -1),
|
||||
"x-phase7i-proxy": "v1",
|
||||
};
|
||||
}
|
||||
|
||||
function mutationHeaders(identity = "alice@example.com"): Record<string, string> {
|
||||
return {
|
||||
...proxyHeaders(identity),
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
origin: PUBLIC_ORIGIN,
|
||||
"sec-fetch-dest": "empty",
|
||||
"sec-fetch-mode": "cors",
|
||||
"sec-fetch-site": "same-origin",
|
||||
};
|
||||
}
|
||||
|
||||
async function withHttpRuntime(
|
||||
run: (context: { runtime: Phase7iDemoRuntime; request: HttpRequest }) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const runtime = new Phase7iDemoRuntime({
|
||||
publicOrigin: PUBLIC_ORIGIN,
|
||||
basePath: "",
|
||||
commitSha: "deadbeef",
|
||||
packageRoot,
|
||||
log: () => undefined,
|
||||
});
|
||||
await runtime.initialize();
|
||||
const server = runtime.createServer();
|
||||
const scratchRoot = process.env.PAPERCLIP_RUN_SCRATCH_DIR ?? tmpdir();
|
||||
const socketDirectory = await mkdtemp(resolve(scratchRoot, "phase7i-http-"));
|
||||
const socketPath = resolve(socketDirectory, "server.sock");
|
||||
await new Promise<void>((resolveListen, rejectListen) => {
|
||||
server.once("error", rejectListen);
|
||||
server.listen(socketPath, () => {
|
||||
server.off("error", rejectListen);
|
||||
resolveListen();
|
||||
});
|
||||
});
|
||||
const request: HttpRequest = async (path, init = {}) => {
|
||||
return await new Promise<HttpResult>((resolveRequest, rejectRequest) => {
|
||||
const request = httpRequest({
|
||||
socketPath,
|
||||
path,
|
||||
method: init.method ?? "GET",
|
||||
headers: init.headers,
|
||||
}, (response) => {
|
||||
const chunks: Buffer[] = [];
|
||||
response.on("data", (chunk: Buffer | string) => {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
});
|
||||
response.once("error", rejectRequest);
|
||||
response.once("end", () => {
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(response.headers)) {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) headers.append(name, item);
|
||||
} else if (value !== undefined) {
|
||||
headers.set(name, String(value));
|
||||
}
|
||||
}
|
||||
const text = Buffer.concat(chunks).toString("utf8");
|
||||
resolveRequest({
|
||||
status: response.statusCode ?? 0,
|
||||
headers,
|
||||
body: text === "" ? null : JSON.parse(text) as unknown,
|
||||
});
|
||||
});
|
||||
});
|
||||
request.once("error", rejectRequest);
|
||||
if (init.body !== undefined) request.write(init.body);
|
||||
request.end();
|
||||
});
|
||||
};
|
||||
try {
|
||||
await run({ runtime, request });
|
||||
} finally {
|
||||
await new Promise<void>((resolveClose, rejectClose) => {
|
||||
server.close((error) => error ? rejectClose(error) : resolveClose());
|
||||
});
|
||||
await runtime.shutdown();
|
||||
await rm(socketDirectory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function createSession(request: HttpRequest, identity: string): Promise<{ id: string; cookie: string }> {
|
||||
const response = await request("/api/phase7/sessions", {
|
||||
method: "POST",
|
||||
headers: mutationHeaders(identity),
|
||||
body: JSON.stringify({ scenarioId: SCENARIO_ID }),
|
||||
});
|
||||
expect(response.status).toBe(201);
|
||||
return { id: sessionId(response), cookie: responseCookie(response) };
|
||||
}
|
||||
|
||||
function sessionId(response: HttpResult): string {
|
||||
if (
|
||||
typeof response.body !== "object" ||
|
||||
response.body === null ||
|
||||
!("sessionId" in response.body) ||
|
||||
typeof response.body.sessionId !== "string"
|
||||
) {
|
||||
throw new Error("response did not contain a session id");
|
||||
}
|
||||
return response.body.sessionId;
|
||||
}
|
||||
|
||||
function responseCookie(response: HttpResult): string {
|
||||
const setCookie = response.headers.get("set-cookie");
|
||||
if (!setCookie) throw new Error("response did not set a capability cookie");
|
||||
return setCookie.split(";", 1)[0]!;
|
||||
}
|
||||
|
||||
async function replaySession(
|
||||
request: HttpRequest,
|
||||
id: string,
|
||||
cookie: string,
|
||||
identity: string,
|
||||
): Promise<HttpResult> {
|
||||
return await request(`/api/phase7/sessions/${id}/replay`, {
|
||||
method: "POST",
|
||||
headers: { ...mutationHeaders(identity), cookie },
|
||||
body: "{}",
|
||||
});
|
||||
}
|
||||
|
||||
function importSpecifiers(source: string): string[] {
|
||||
const matches = source.matchAll(/\b(?:import|export)\s+(?:type\s+)?(?:[^"'`;]{0,500}?\s+from\s+)?["']([^"']+)["']/g);
|
||||
return [...matches].map((match) => match[1]!);
|
||||
|
||||
@@ -159,7 +159,8 @@ function sendError(res: ServerResponse, error: RequestFailure): void {
|
||||
}
|
||||
|
||||
async function readJsonBody(req: IncomingMessage): Promise<Record<string, unknown>> {
|
||||
if (!header(req, "content-type").toLowerCase().startsWith("application/json")) {
|
||||
const mediaType = header(req, "content-type").split(";", 1)[0]?.trim().toLowerCase();
|
||||
if (mediaType !== "application/json") {
|
||||
throw new RequestFailure(415, "json_required", "Mutation requests require application/json.");
|
||||
}
|
||||
const declaredLength = Number(header(req, "content-length"));
|
||||
@@ -359,11 +360,11 @@ export class Phase7iDemoRuntime {
|
||||
if (header(req, "x-phase7i-proxy") !== "v1") {
|
||||
throw new RequestFailure(403, "trusted_proxy_required", "Request did not arrive through the trusted proxy.");
|
||||
}
|
||||
const forwardedProtocol = header(req, "x-forwarded-proto").split(",")[0]?.trim().toLowerCase();
|
||||
const forwardedProtocol = header(req, "x-forwarded-proto").toLowerCase();
|
||||
if (forwardedProtocol !== this.publicOrigin.protocol.slice(0, -1)) {
|
||||
throw new RequestFailure(403, "transport_denied", "Forwarded transport is not allowed.");
|
||||
}
|
||||
const forwardedHost = header(req, "x-forwarded-host").split(",")[0]?.trim().toLowerCase();
|
||||
const forwardedHost = header(req, "x-forwarded-host").toLowerCase();
|
||||
if (forwardedHost !== this.publicOrigin.host.toLowerCase()) {
|
||||
throw new RequestFailure(403, "host_denied", "Forwarded host is not allowed.");
|
||||
}
|
||||
@@ -374,9 +375,12 @@ export class Phase7iDemoRuntime {
|
||||
if (header(req, "origin") !== this.publicOrigin.origin) {
|
||||
throw new RequestFailure(403, "origin_denied", "Request origin is not allowed.");
|
||||
}
|
||||
const fetchSite = header(req, "sec-fetch-site").toLowerCase();
|
||||
if (fetchSite && fetchSite !== "same-origin" && fetchSite !== "none") {
|
||||
throw new RequestFailure(403, "fetch_metadata_denied", "Cross-site mutation requests are not allowed.");
|
||||
if (
|
||||
header(req, "sec-fetch-site").toLowerCase() !== "same-origin" ||
|
||||
header(req, "sec-fetch-mode").toLowerCase() !== "cors" ||
|
||||
header(req, "sec-fetch-dest").toLowerCase() !== "empty"
|
||||
) {
|
||||
throw new RequestFailure(403, "fetch_metadata_denied", "Mutation request metadata is not allowed.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -257,17 +257,6 @@ describe("Phase 7 exposure and authorization", () => {
|
||||
});
|
||||
expect(revisions).toMatchObject({ ok: true, value: [{ id: revisionId, revision: 1 }] });
|
||||
|
||||
expect(await runtime.invoke({
|
||||
operationId: "request_human_input",
|
||||
input: {
|
||||
interactionKind: "confirmation",
|
||||
title: "Confirm plan",
|
||||
prompt: "Accept this plan?",
|
||||
targetRevisionId: revisionId!,
|
||||
continuationPolicy: "wake_assignee",
|
||||
},
|
||||
idempotencyKey: "confirm-plan-1",
|
||||
})).toMatchObject({ ok: true });
|
||||
expect(await runtime.invoke({
|
||||
operationId: "register_deliverable",
|
||||
input: {
|
||||
@@ -285,6 +274,17 @@ describe("Phase 7 exposure and authorization", () => {
|
||||
input: { summary: "Ready for semantic review." },
|
||||
idempotencyKey: "review-1",
|
||||
})).toMatchObject({ ok: true });
|
||||
expect(await runtime.invoke({
|
||||
operationId: "request_human_input",
|
||||
input: {
|
||||
interactionKind: "confirmation",
|
||||
title: "Confirm plan",
|
||||
prompt: "Accept this plan?",
|
||||
targetRevisionId: revisionId!,
|
||||
continuationPolicy: "wake_assignee",
|
||||
},
|
||||
idempotencyKey: "confirm-plan-1",
|
||||
})).toMatchObject({ ok: true });
|
||||
|
||||
expect(adapter.snapshot()).toMatchObject({
|
||||
tasks: [{ status: "in_review" }],
|
||||
@@ -294,6 +294,34 @@ describe("Phase 7 exposure and authorization", () => {
|
||||
workProducts: [{ type: "artifact" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a pending human-input handoff in review instead of weakening the lifecycle", async () => {
|
||||
const { adapter, runtime } = await runtimeFor();
|
||||
expect(await runtime.invoke({
|
||||
operationId: "request_human_input",
|
||||
input: {
|
||||
interactionKind: "confirmation",
|
||||
title: "Confirm plan",
|
||||
prompt: "Accept this plan?",
|
||||
continuationPolicy: "wake_assignee",
|
||||
},
|
||||
idempotencyKey: "confirm-plan-first",
|
||||
})).toMatchObject({ ok: true });
|
||||
|
||||
expect(await runtime.invoke({
|
||||
operationId: "request_review",
|
||||
input: { summary: "A duplicate review transition must fail closed." },
|
||||
idempotencyKey: "duplicate-review",
|
||||
})).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "operation_unsupported", reason: "mock_operation_rejected" },
|
||||
});
|
||||
expect(adapter.snapshot()).toMatchObject({
|
||||
tasks: [{ status: "in_review" }],
|
||||
interactions: [{ status: "pending" }],
|
||||
comments: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Phase 7 security policy", () => {
|
||||
|
||||
Reference in new issue
Block a user