From 53a0cf3b0cad39eb833accde40350cecd48c2f6f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 11 Sep 2026 15:38:31 -0500 Subject: [PATCH] Tolerate PostgreSQL PID removal during post-run secret scanning --- tests/runner-e2e/launch.ts | 3 ++ tests/runner-e2e/redaction.ts | 30 ++++++++++---- tests/runner-e2e/support.test.ts | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 450b68cc356..5c59fd24cf5 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -33,6 +33,7 @@ import { assertSecretFree, findSecretLeakInDirectory, isEphemeralCodexRuntimeAuthFile, + isEphemeralPostgresPidFile, normalizedSecrets, sanitizeJson, } from "./redaction.js"; @@ -789,6 +790,8 @@ async function runAttempt(input: { const leak = await findSecretLeakInDirectory(directory, credentials, { includeShapes: false, ignoreFile: (file) => expectedEphemeralCredentials.has(file), + allowDisappearedFile: (file) => + label === "Paperclip home" && isEphemeralPostgresPidFile(paperclipHome, file), }); if (!leak) break; const isManagedCodexRuntimeAuth = diff --git a/tests/runner-e2e/redaction.ts b/tests/runner-e2e/redaction.ts index d1214542c67..fa7a89eaf35 100644 --- a/tests/runner-e2e/redaction.ts +++ b/tests/runner-e2e/redaction.ts @@ -152,12 +152,18 @@ export function assertSecretFree( if (leak) throw new Error(`Secret leak in ${label}: ${leak}`); } +export function isEphemeralPostgresPidFile(paperclipHome: string, file: string): boolean { + const relative = path.relative(paperclipHome, file).split(path.sep).join("/"); + return /^instances\/[^/]+\/db\/postmaster\.pid$/.test(relative); +} + export async function findSecretLeakInDirectory( root: string, secrets: readonly string[], options: { includeShapes?: boolean; ignoreFile?: (file: string) => boolean; + allowDisappearedFile?: (file: string) => boolean; } = {}, ): Promise<{ file: string; reason: string } | null> { const overlap = Math.max( @@ -179,14 +185,22 @@ export async function findSecretLeakInDirectory( } else if (entry.isFile()) { if (options.ignoreFile?.(file)) continue; let carry = Buffer.alloc(0); - for await (const chunk of createReadStream(file)) { - const data = Buffer.concat([ - carry, - Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk), - ]); - const reason = findSecretLeak(data, secrets, options); - if (reason) return { file, reason }; - carry = data.subarray(Math.max(0, data.length - overlap)); + try { + for await (const chunk of createReadStream(file)) { + const data = Buffer.concat([ + carry, + Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk), + ]); + const reason = findSecretLeak(data, secrets, options); + if (reason) return { file, reason }; + carry = data.subarray(Math.max(0, data.length - overlap)); + } + } catch (error) { + // PostgreSQL removes its PID file on shutdown, possibly after readdir. + // Existing contents are still scanned; only the caller's exact + // transient paths may disappear. Evidence and other I/O errors fail. + if ((error as NodeJS.ErrnoException)?.code !== "ENOENT" || + !options.allowDisappearedFile?.(file)) throw error; } } } diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index 735a2884191..d4ad4bac35c 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -1,4 +1,5 @@ import { execFileSync } from "node:child_process"; +import { mkdirSync, unlinkSync } from "node:fs"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -24,6 +25,7 @@ import { findSecretLeakInJsonValues, findSecretLeakInDirectory, isEphemeralCodexRuntimeAuthFile, + isEphemeralPostgresPidFile, redactText, sanitizeJson, } from "./redaction.js"; @@ -959,6 +961,74 @@ describe("runner E2E evidence redaction", () => { ).resolves.toMatchObject({ reason: "exact secret value" }); }); + it("tolerates only the embedded PostgreSQL PID disappearing during shutdown and keeps scanning", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-postgres-pid-race-")); + cleanupDirectories.push(root); + const pidFile = path.join(root, "instances", "test", "db", "postmaster.pid"); + const persistedFile = path.join(path.dirname(pidFile), "z-persisted.bin"); + await mkdir(path.dirname(pidFile), { recursive: true }); + await writeFile(pidFile, "1234\n"); + await writeFile(persistedFile, secret); + await expect(findSecretLeakInDirectory(root, [secret], { + ignoreFile: (file) => { + if (file === pidFile) unlinkSync(file); // Removed after directory enumeration. + return false; + }, + allowDisappearedFile: (file) => isEphemeralPostgresPidFile(root, file), + })).resolves.toEqual({ file: persistedFile, reason: "exact secret value" }); + expect(isEphemeralPostgresPidFile(root, path.join(root, "workspace", "postmaster.pid"))).toBe(false); + expect(isEphemeralPostgresPidFile(root, path.join(root, "instances", "test", "db", "records.bin"))).toBe(false); + }); + + it("still detects secrets in an existing PostgreSQL PID file", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-postgres-pid-secret-")); + cleanupDirectories.push(root); + const pidFile = path.join(root, "instances", "test", "db", "postmaster.pid"); + await mkdir(path.dirname(pidFile), { recursive: true }); + await writeFile(pidFile, secret); + await expect(findSecretLeakInDirectory(root, [secret], { + allowDisappearedFile: (file) => isEphemeralPostgresPidFile(root, file), + })).resolves.toEqual({ file: pidFile, reason: "exact secret value" }); + }); + + it("fails when required persisted state disappears during scanning", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-required-file-race-")); + cleanupDirectories.push(root); + const requiredFile = path.join(root, "records.json"); + await writeFile(requiredFile, "{}"); + await expect(findSecretLeakInDirectory(root, [secret], { + ignoreFile: (file) => { unlinkSync(file); return false; }, + allowDisappearedFile: (file) => isEphemeralPostgresPidFile(root, file), + })).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("does not suppress other I/O failures for the ephemeral PID path", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-postgres-pid-io-")); + cleanupDirectories.push(root); + const pidFile = path.join(root, "instances", "test", "db", "postmaster.pid"); + await mkdir(path.dirname(pidFile), { recursive: true }); + await writeFile(pidFile, "1234\n"); + await expect(findSecretLeakInDirectory(root, [secret], { + ignoreFile: (file) => { unlinkSync(file); mkdirSync(file); return false; }, + allowDisappearedFile: (file) => isEphemeralPostgresPidFile(root, file), + })).rejects.toMatchObject({ code: "EISDIR" }); + }); + + it("still reports missing mandatory pass evidence", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-required-evidence-")); + cleanupDirectories.push(root); + const privateDir = path.join(root, "private"); + await mkdir(privateDir); + const packaged = await packageEvidence({ + privateDir, + uploadDir: path.join(root, "upload"), + secrets: [secret], + expectPassScreenshot: true, + }); + expect(packaged.missing).toContain("result.json"); + expect(packaged.missing).toContain("final-state.png"); + }); + it("can ignore fake key shapes while scanning persisted package state", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-secret-shape-test-"),