mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(evals): give choose-capable-target a saveable computer, grade it structurally and surface setup validation reasons
The fixture saved a built-in fake sandbox as its unsupported control. The product has refused to save fake sandboxes since #4449 ("reserved for internal probes"), so setup returned 422 before any agent run, and the test API withheld the reason. No saveable computer is advertised as unable to run Claude Code, so the case now contrasts advertised support with runtime health: the fixture saves an SSH computer on a reserved .invalid host that is advertised for Claude Code but never tested. The yaml context records this premise change (manifest changedFields pinned). The grader no longer reads reply keywords. It requires the agent boundary (403 on capabilities and probe), a rendered reply link to the environment settings page that the person then clicks to see both computers and the checked capabilities, no board-only SSH settings in the reply, unchanged targets and worker, and exactly one chat turn. Cookie-session API failures with 400/409/422 now report the product's own validation message, bounded and with request values and the session redacted. Details and other statuses stay withheld. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e5d7678e4d
commit
fa50b8d662
9 files changed
+194
-92
No files matched your search
+33
-1
@@ -52,6 +52,34 @@ export class RemoteAdmissionReadError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** Leaf strings of a request body: a validation message that echoes one is redacted. */
|
||||
function requestStrings(value: unknown, out: string[] = []): string[] {
|
||||
if (typeof value === "string") { if (value.length >= 8) out.push(value); }
|
||||
else if (Array.isArray(value)) for (const item of value) requestStrings(item, out);
|
||||
else if (value && typeof value === "object") for (const item of Object.values(value)) requestStrings(item, out);
|
||||
return out;
|
||||
}
|
||||
/** The product's own validation message from a 400/409/422 body, bounded and redacted.
|
||||
* Details, issues and any other field stay withheld; request values and the session never appear. */
|
||||
async function validationReason(response: Response, request: unknown, session?: string): Promise<string | undefined> {
|
||||
const reader = response.body?.getReader(); if (!reader) return;
|
||||
const chunks: Uint8Array[] = []; let bytes = 0, done = false;
|
||||
try {
|
||||
for (;;) {
|
||||
const next = await reader.read(); if (next.done) { done = true; break; }
|
||||
bytes += next.value.byteLength; if (bytes > 16_384) return;
|
||||
chunks.push(next.value);
|
||||
}
|
||||
const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { error?: unknown };
|
||||
if (!body || typeof body !== "object" || typeof body.error !== "string") return;
|
||||
let reason = body.error.replace(/\s+/g, " ").trim();
|
||||
for (const secret of [...requestStrings(request), ...(session ? [session] : [])].sort((a, b) => b.length - a.length)) reason = reason.split(secret).join("[redacted]");
|
||||
reason = reason.replace(/[A-Za-z0-9+/_=-]{32,}/g, "[redacted]");
|
||||
return reason ? (reason.length > 240 ? `${reason.slice(0, 240)}...` : reason) : undefined;
|
||||
} catch { return; }
|
||||
finally { if (!done) await reader.cancel().catch(() => {}); reader.releaseLock(); }
|
||||
}
|
||||
|
||||
export class RunnerApi {
|
||||
readonly baseURL: string;
|
||||
private browserCookie?: string;
|
||||
@@ -232,7 +260,11 @@ export class RunnerApi {
|
||||
private async authenticatedJson<T>(method: string, path: string, data?: unknown, options?: { timeout: number }): Promise<T> {
|
||||
const response = await this.authenticatedResponse(method, path, data, options);
|
||||
const endpoint = new URL(path, this.baseURL).pathname;
|
||||
if (!response.ok) { await response.body?.cancel(); throw new RunnerApiHttpError(response.status, `${method} ${endpoint} returned ${response.status}; body withheld`); }
|
||||
if (!response.ok) {
|
||||
const reason = [400, 409, 422].includes(response.status) ? await validationReason(response, data, this.browserCookie) : undefined;
|
||||
if (!reason) await response.body?.cancel();
|
||||
throw new RunnerApiHttpError(response.status, `${method} ${endpoint} returned ${response.status}; ${reason ? `reason: ${reason}; other body fields withheld` : "body withheld"}`);
|
||||
}
|
||||
try { return await response.json() as T; } catch { throw new Error(`${method} ${endpoint} returned ${response.status} with invalid JSON; body withheld`); }
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user