Files
PaperClipAI/tests/runner-e2e/api.ts
T
DottaandPaperclip 8c9dd1e444 test(runner): qualify Kimi Code lifecycle and recovery (#18029)
Add the Kimi lifecycle and recovery qualification harness. Preserve original grades and cleanup evidence, strict native ownership audits, bounded startup and cleanup, and early rejection of unsupported local fault execution before credentials or resources.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-30 22:54:37 -05:00

180 lines
7.1 KiB
TypeScript

import type { APIRequestContext, APIResponse } from "@playwright/test";
export interface JsonRecord {
[key: string]: unknown;
}
async function failureMessage(response: APIResponse, method: string) {
const text = await response.text().catch(() => "");
return `${method} ${response.url()} returned ${response.status()}${text ? `: ${text}` : ""}`;
}
/** Preserve HTTP status independently of diagnostic response text. */
export class RunnerApiHttpError extends Error {
constructor(readonly status: number, message: string) { super(message); this.name = "RunnerApiHttpError"; }
}
/** Admission-only diagnostics. Never retain the rejected value or its cause. */
export class RemoteAdmissionReadError extends Error {
constructor(
readonly endpoint: "issue" | "run" | "leases",
readonly failureClass: "candidate_failure" | "transient_infrastructure" | "permanent_infrastructure",
) {
super(`Remote admission ${endpoint} read failed (${failureClass}); diagnostics withheld`);
this.name = "RemoteAdmissionReadError";
}
}
export class RunnerApi {
readonly baseURL: string;
private browserCookie?: string;
setBrowserSession(cookie: string) { this.browserCookie = cookie; }
private sessionOptions() {
return this.browserCookie ? { headers: { Cookie: this.browserCookie, Origin: this.baseURL } } : {};
}
constructor(readonly request: APIRequestContext) {
const port = process.env.PAPERCLIP_RUNNER_E2E_PORT?.trim();
if (!port) throw new Error("PAPERCLIP_RUNNER_E2E_PORT is required");
this.baseURL = `http://127.0.0.1:${port}`;
}
async get<T>(path: string, options?: { timeout: number }): Promise<T> {
const response = await this.request.get(path, { ...this.sessionOptions(), ...options });
if (!response.ok()) throw new RunnerApiHttpError(response.status(), await failureMessage(response, "GET"));
return response.json() as Promise<T>;
}
async bytes(path: string): Promise<Buffer> {
const response = await this.request.get(path, this.sessionOptions());
if (!response.ok()) throw new RunnerApiHttpError(response.status(), await failureMessage(response, "GET"));
return response.body();
}
async post<T>(path: string, data?: unknown, options?: { timeout: number }): Promise<T> {
const response = await this.request.post(path, { data, ...this.sessionOptions(), ...options });
if (!response.ok()) throw new Error(await failureMessage(response, "POST"));
return response.json() as Promise<T>;
}
/**
* Playwright traces APIRequestContext request bodies. Secret creation must
* still use the public API, but it goes through Node fetch so plaintext is
* never serialized into trace/blob evidence before Paperclip encrypts it.
*/
async postSensitive<T>(path: string, data: unknown): Promise<T> {
return this.sensitive<T>("POST", path, data);
}
async putSensitive<T>(path: string, data: unknown): Promise<T> {
return this.sensitive<T>("PUT", path, data);
}
private async sensitive<T>(method: "POST" | "PUT", path: string, data: unknown): Promise<T> {
const url = new URL(path, this.baseURL);
if (url.origin !== this.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Sensitive request must use the fixture API");
let response: Response;
try {
response = await fetch(url, {
method, redirect: "error", signal: AbortSignal.timeout(15_000),
headers: { "content-type": "application/json", ...this.sessionOptions().headers },
body: JSON.stringify(data),
});
} catch { throw new Error(`Sensitive ${method} request failed; details withheld`); }
if (!response.ok) {
await response.body?.cancel();
throw new Error(`Sensitive ${method} ${url.pathname} returned ${response.status}; response body withheld`);
}
return response.json() as Promise<T>;
}
async patch<T>(path: string, data: unknown): Promise<T> {
const response = await this.request.patch(path, { data, ...this.sessionOptions() });
if (!response.ok())
throw new Error(await failureMessage(response, "PATCH"));
return response.json() as Promise<T>;
}
/** Use the public revision fence when configuring a fixture's instruction entry. */
async saveAgentInstructions(agentId: string, content: string): Promise<void> {
const path = `/api/agents/${agentId}/instructions-bundle/file`;
const current = await this.request.get(`${path}?path=AGENTS.md`);
let baseHash: string | null = null;
if (current.ok()) {
const detail = await current.json();
if (typeof detail.contentHash !== "string" || !detail.contentHash) {
throw new Error("Existing fixture instructions have no revision hash");
}
baseHash = detail.contentHash;
} else if (current.status() !== 404) {
throw new Error(await failureMessage(current, "GET"));
}
const saved = await this.request.put(path, { data: { path: "AGENTS.md", content, baseHash } });
if (!saved.ok()) throw new Error(await failureMessage(saved, "PUT"));
}
async delete(
path: string,
options?: { allowNotFound?: boolean; timeout?: number },
): Promise<void> {
const response = await this.request.delete(path, { ...this.sessionOptions(), timeout: options?.timeout });
if (response.ok() || (options?.allowNotFound && response.status() === 404))
return;
throw new Error(await failureMessage(response, "DELETE"));
}
}
export class ObservedStateTimeout extends Error {
constructor(label: string, readonly failureClass: "candidate_failure" | "transient_infrastructure", detail?: string) {
super(`Timed out waiting for ${label}; ${detail ?? "the observed state did not satisfy the condition"}. See the saved state evidence.`);
this.name = "ObservedStateTimeout";
}
}
export async function pollUntil<T>(input: {
label: string;
deadlineAt: number;
load: () => Promise<T>;
accept: (value: T) => boolean;
reject?: (value: T) => string | undefined;
intervalMs?: number;
retryOnError?: (error: unknown) => boolean;
timeoutFailureClass?: "candidate_failure" | "transient_infrastructure";
timeoutDetail?: (value: T | undefined) => string | undefined;
}): Promise<T> {
let last: T | undefined;
let lastError: unknown;
while (Date.now() < input.deadlineAt) {
try {
last = await input.load();
if (input.accept(last)) return last;
const rejection = input.reject?.(last);
if (rejection) {
throw new Error(`Stopped waiting for ${input.label}: ${rejection}`);
}
lastError = undefined;
} catch (error) {
if (input.retryOnError && !input.retryOnError(error)) throw error;
if (
error instanceof Error &&
error.message.startsWith(`Stopped waiting for ${input.label}:`)
) {
throw error;
}
lastError = error;
}
await new Promise((resolve) =>
setTimeout(resolve, input.intervalMs ?? 2_000),
);
}
if (lastError instanceof Error) {
throw new Error(`Timed out waiting for ${input.label}: ${lastError.message}`, { cause: lastError });
}
throw new ObservedStateTimeout(
input.label,
input.timeoutFailureClass ?? "candidate_failure",
input.timeoutDetail?.(last),
);
}