From 5730403309d193c88a005b4e5813fe9cf3c23184 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 13:47:09 -0600 Subject: [PATCH] test(evals): name Node error codes and types; read memfd-launched opencode serve Co-Authored-By: Claude Opus 5.5 --- .../chat-catalog-domains/runtime-provider-cleanup.ts | 5 +++-- tests/runner-e2e/private-attempt-diagnostic.test.ts | 7 +++++++ tests/runner-e2e/private-attempt-diagnostic.ts | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts b/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts index 96e84f22576..729a361db2a 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts @@ -31,7 +31,8 @@ async function providerIdentity(pid: number): Promise { const [uid, parent, executable, command, startedAt] = await Promise.all([ps("uid"), ps("ppid"), ps("comm"), ps("args"), readProcessStartedAt(pid)]); // A pinned binary may be executed from an inherited descriptor (/proc/self/fd/N); its real // executable is then only reachable through the kernel's /proc//exe link. - const canonical = await realpath(ownedNativeExecutablePath(pid, executable)).catch(() => realpath(`/proc/${pid}/exe`)); + // An in-memory (memfd) binary has no resolvable path, but the kernel link still reads its bytes. + const canonical = await realpath(ownedNativeExecutablePath(pid, executable)).catch(() => realpath(`/proc/${pid}/exe`)).catch(() => `/proc/${pid}/exe`); const args = command.split(/\s+/); // node may carry runtime flags before the script; the absolute proxy script is the last argument. const script = args.at(-1)!, flags = args.slice(1, -1); @@ -41,7 +42,7 @@ async function providerIdentity(pid: number): Promise { return { pid, parentPid: Number(parent), uid: Number(uid), startedAt, executable: canonical, digest: createHash("sha256").update(await readFile(canonical)).digest("hex"), appServer: /(?:^|\s)app-server(?=\s|$)/.test(command), // `opencode serve` runs either as its own binary or as node running the opencode launcher script. - openCodeServe: args.includes("serve") && (/^opencode/.test(path.basename(canonical)) + openCodeServe: args.includes("serve") && (/^opencode/.test(path.basename(canonical)) || /^\/(?:proc\/self|dev)\/fd\/\d+$/.test(args[0] ?? "") || args.slice(1, args.indexOf("serve")).some(arg => path.isAbsolute(arg) && /^opencode(?:\.[cm]?js)?$/.test(path.basename(arg)))), proxyScript: proxy, proxyDigest }; } catch { throw new Error("Provider cleanup fixture could not qualify the exact owned process; private diagnostics withheld"); } } diff --git a/tests/runner-e2e/private-attempt-diagnostic.test.ts b/tests/runner-e2e/private-attempt-diagnostic.test.ts index b46a4144189..71df68dc0a1 100644 --- a/tests/runner-e2e/private-attempt-diagnostic.test.ts +++ b/tests/runner-e2e/private-attempt-diagnostic.test.ts @@ -136,3 +136,10 @@ it("names other filesystem refusals by their closed error code", () => { expect(privateAttemptDiagnostic({ stage: "seal-original", error }).finalization).toBe("fs-EEXIST"); expect(JSON.stringify(privateAttemptDiagnostic({ stage: "seal-original", error }))).not.toContain("/private/path"); }); +it("names Node error codes, error types and non-error throws without their messages", () => { + const node = Object.assign(new Error("File size (3) is greater than 2 GiB: /private/x"), { code: "ERR_FS_FILE_TOO_LARGE" }); + expect(privateAttemptDiagnostic({ stage: "attach-original", error: node }).finalization).toBe("fs-ERR_FS_FILE_TOO_LARGE"); + expect(privateAttemptDiagnostic({ stage: "attach-original", error: new TypeError("x of /private/y") }).finalization).toBe("error-type-TypeError"); + expect(privateAttemptDiagnostic({ stage: "attach-original", error: "raw /private/z" }).finalization).toBe("non-error-thrown"); + expect(JSON.stringify(privateAttemptDiagnostic({ stage: "attach-original", error: node }))).not.toContain("/private/"); +}); diff --git a/tests/runner-e2e/private-attempt-diagnostic.ts b/tests/runner-e2e/private-attempt-diagnostic.ts index 291785c855f..da20a3f3a17 100644 --- a/tests/runner-e2e/private-attempt-diagnostic.ts +++ b/tests/runner-e2e/private-attempt-diagnostic.ts @@ -195,7 +195,9 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) { status: member(result.status, ["passed", "failed", "skipped"]), failureClass: member(result.failureClass, ["candidate_failure", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]), finalization: failures.get(message) ?? (record(input.error).code === "ENOENT" ? "missing-file" - : typeof record(input.error).code === "string" && /^E[A-Z]{2,15}$/.test(record(input.error).code) ? `fs-${record(input.error).code}` + : typeof record(input.error).code === "string" && /^(?:E[A-Z]{2,15}|ERR_[A-Z0-9_]{2,48})$/.test(record(input.error).code) ? `fs-${record(input.error).code}` + : input.error !== undefined && !(input.error instanceof Error) ? "non-error-thrown" + : input.error instanceof Error && input.error.name !== "Error" && /^[A-Z][A-Za-z]{2,40}$/.test(input.error.name) ? `error-type-${input.error.name}` : /^Secret leak in [\s\S]*: (?:exact secret value|secret-shaped value)$/.test(message) ? "text-secret-finding" : "unclassified"), phase: phases.find(([label]) => [...errors, message].some(error => error.includes(label!)))?.[1] ?? "unclassified", privacy: { persistedState: errors.some(error => error.startsWith("Secret leak in persisted ")),