From 4e73b7bb70fa619f379c6665457d66d4f6d57180 Mon Sep 17 00:00:00 2001
From: Dotta <bippadotta@protonmail.com>
Date: Mon, 26 Oct 2026 21:18:45 -0500
Subject: [PATCH] test(evals): protect short multiline diagnostic credentials

---
 tests/runner-e2e/private-diagnostics.test.ts |  8 ++++----
 tests/runner-e2e/private-diagnostics.ts      | 10 ++++++++--
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/runner-e2e/private-diagnostics.test.ts b/tests/runner-e2e/private-diagnostics.test.ts
index 9f83329f0f2..f907afad33d 100644
--- a/tests/runner-e2e/private-diagnostics.test.ts
+++ b/tests/runner-e2e/private-diagnostics.test.ts
@@ -9,11 +9,11 @@ function registry() { const root = realpathSync(mkdtempSync(path.join(os.tmpdir(
 describe("private diagnostic output boundary", () => {
   it("redacts secret and escaped variants across chunks, multiline PEM and UTF-8 without losing ordinary failure detail", () => {
     const password = 'synthetic-credential-<&"-12345678';
-    const pem = "-----BEGIN PRIVATE KEY-----\nU1lOVEhFVElDLVByaXZhdGUtS2V5LUxpbmUx\nU1lOVEhFVElDLVByaXZhdGUtS2V5LUxpbmUy\n-----END PRIVATE KEY-----";
-    const outputs: string[] = []; const stream = createPrivateDiagnosticStream(() => [password, pem], value => outputs.push(value));
-    const input = `HTTP503: π provider unavailable ${password}\n${pem}\nJSON ${JSON.stringify(password)}\nHTML ${password.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/"/g, "&quot;")}\nURI ${encodeURIComponent(password)}\n`;
+    const pem = "-----BEGIN PRIVATE KEY-----\nU1lOVEhFVElDLVByaXZhdGUtS2V5LUxpbmUx\nU1lOVEhFVElDLVByaXZhdGUtS2V5LUxpbmUy\n-----END PRIVATE KEY-----"; const short = "Ax4\nBy5\nCz6";
+    const outputs: string[] = []; const stream = createPrivateDiagnosticStream(() => [password, pem, short], value => outputs.push(value));
+    const input = `HTTP503: π provider unavailable ${password}\n${pem}\n${short}\nJSON ${JSON.stringify(password)}\nHTML ${password.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/"/g, "&quot;")}\nURI ${encodeURIComponent(password)}\n`;
     const bytes = Buffer.from(input); for (let i = 0; i < bytes.length; i += 3) stream.write(bytes.subarray(i, i + 3)); stream.end();
-    const safe = outputs.join(""); for (const value of diagnosticSecrets([password, pem])) expect(safe).not.toContain(value);
+    const safe = outputs.join(""); for (const value of diagnosticSecrets([password, pem, short])) expect(safe).not.toContain(value);
     expect(safe).toContain("HTTP503: π provider unavailable"); expect(stream.redactedSecret()).toBe(true); expect(stream.failed()).toBe(false);
   });
   it("synchronously registers a generated cookie before first output and rereads at final flush", () => {
diff --git a/tests/runner-e2e/private-diagnostics.ts b/tests/runner-e2e/private-diagnostics.ts
index 0332bf4a1b5..c1cb46aa64a 100644
--- a/tests/runner-e2e/private-diagnostics.ts
+++ b/tests/runner-e2e/private-diagnostics.ts
@@ -46,9 +46,15 @@ export function trackedPrivateSecrets(initial: string[], executionId: string, en
   } });
 }
 export function diagnosticSecrets(values: readonly string[]): string[] {
-  const variants = normalizedSecrets(values).flatMap(value => [value, JSON.stringify(value).slice(1, -1), encodeURIComponent(value),
+  const variants = normalizedSecrets(values).flatMap(value => {
+    let structured = false; try { const parsed = JSON.parse(value); structured = parsed !== null && typeof parsed === "object"; } catch { /* Ordinary secret. */ }
+    return [value, JSON.stringify(value).slice(1, -1), encodeURIComponent(value),
     value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;"),
-    ...value.split(/\r?\n/).filter(line => line.length >= 8)]);
+    // Short nonempty password/key lines are private too. JSON punctuation is
+    // structural; its actual string credential values are separately indexed
+    // by normalizedSecrets, and longer serialized field lines remain covered.
+    ...value.split(/\r?\n/).filter(line => structured ? line.length >= 8 : line.length > 0)];
+  });
   return normalizedSecrets(variants);
 }
 /** No raw byte reaches a console or log. UTF-8 and secret boundaries may span