From a40043db19b4ff1c0c53af2d20b5bf0c28fd5268 Mon Sep 17 00:00:00 2001 From: Tonio Date: Thu, 27 Aug 2026 09:08:03 -0700 Subject: [PATCH] fix(smoke): ride out a cold auth stack in the release bootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly release smoke has been flaking for a week — failing 2026-08-21, -23, -24, -26 and -27 while passing on the days between, on identical inputs. Each failure was `could not sign up or sign in admin user`, and because publish_nightly needs smoke, every one of those days published no nightly and the release train stalled. `/api/health` answering 200 is not a promise that auth can serve a write. The container cold-installs paperclipai and brings up embedded postgres, and there is a window where health answers while a sign-up still fails. The bootstrap made exactly one sign-up attempt and one sign-in attempt inside that window, so whether the release train moved came down to timing. Retry the pair until a deadline. Sign-in already covers a data dir that holds the admin, including one created by an earlier pass, so repeating the pair is safe. Also print the HTTP status on failure. It was in hand the whole time and never surfaced: the message printed the response body, and a sign-up that could not connect has none, so a week of failures produced "Sign-up response:" followed by a blank line and nothing to act on. Tested without Docker by driving the real functions out of the script with a stubbed request helper — extracted rather than restated, so the test cannot drift from what CI runs. Co-Authored-By: Claude Fable 5 --- .../docker-onboard-smoke-bootstrap.test.mjs | 151 ++++++++++++++++++ scripts/docker-onboard-smoke.sh | 54 ++++++- 2 files changed, 198 insertions(+), 7 deletions(-) create mode 100644 scripts/__tests__/docker-onboard-smoke-bootstrap.test.mjs diff --git a/scripts/__tests__/docker-onboard-smoke-bootstrap.test.mjs b/scripts/__tests__/docker-onboard-smoke-bootstrap.test.mjs new file mode 100644 index 00000000000..b0bf2e97577 --- /dev/null +++ b/scripts/__tests__/docker-onboard-smoke-bootstrap.test.mjs @@ -0,0 +1,151 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const script = new URL("../docker-onboard-smoke.sh", import.meta.url).pathname; + +/** + * The admin bootstrap in `docker-onboard-smoke.sh`, exercised without Docker. + * + * `/api/health` answering 200 is not a promise that auth can serve a write: the + * container cold-installs paperclipai and brings up embedded postgres, and there + * is a window where health answers while a sign-up still fails. A single + * un-retried attempt in that window is what made the nightly release smoke flake + * — it failed on 2026-08-21, -23, -24, -26 and -27 while passing on the days + * between, on identical inputs. + * + * These tests drive the real functions out of the shipped script with a stubbed + * request helper. Extracting them rather than restating them keeps the test + * honest: a copy would drift from what actually runs in CI. + */ + +const source = fs.readFileSync(script, "utf8"); + +function extractFunction(name) { + // Anchored to the start of a line: a bare indexOf for `sign_up_or_sign_in() {` + // also matches inside `try_sign_up_or_sign_in() {`, which is defined first. + const marker = `\n${name}() {`; + const at = source.indexOf(marker); + assert.notEqual(at, -1, `${name} not found at column 0 in docker-onboard-smoke.sh`); + const start = at + 1; + const end = source.indexOf("\n}\n", start); + assert.notEqual(end, -1, `${name} has no closing brace at column 0`); + return source.slice(start, end + 3); +} + +/** + * Runs the bootstrap against a scripted sequence of responses. + * + * Each entry in `attempts` is `[signUpStatus, signInStatus]` for one pass of the + * loop; "000" stands for curl failing to connect at all, which is what the + * server returns while it is still coming up — and which produces the empty + * body the real failures showed. + */ +function runBootstrap(attempts, { retrySeconds = 6 } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "smoke-bootstrap-")); + try { + const table = attempts.map((pair) => pair.join(" ")).join("\n"); + fs.writeFileSync(path.join(dir, "responses"), `${table}\n`); + + // The helper is called as `$(...)`, so a counter held in a shell variable + // would die with that subshell. It lives in a file for the same reason. + const harness = ` +set -uo pipefail +TMP_DIR=${JSON.stringify(dir)} +COUNTER="$TMP_DIR/attempt" +echo 0 > "$COUNTER" +PAPERCLIP_PUBLIC_URL="http://stub" +SMOKE_ADMIN_NAME="Smoke Admin" +SMOKE_ADMIN_EMAIL="smoke-admin@paperclip.local" +SMOKE_ADMIN_PASSWORD="pw" +SMOKE_BOOTSTRAP_RETRY_SECONDS=${retrySeconds} + +post_json_with_cookies() { + local url="$1" body="$2" out="$3" + local idx line code + idx="$(cat "$COUNTER")" + line="$(sed -n "$((idx + 1))p" "$TMP_DIR/responses")" + [[ -z "$line" ]] && line="000 000" + if [[ "$url" == *sign-up* ]]; then + code="\${line%% *}" + [[ "$code" =~ ^2 ]] && echo $((idx + 1)) > "$COUNTER" + else + code="\${line##* }" + echo $((idx + 1)) > "$COUNTER" + fi + if [[ "$code" == "000" ]]; then : > "$out"; else echo "{\\"stub\\":$code}" > "$out"; fi + printf '%s' "$code" +} + +${extractFunction("try_sign_up_or_sign_in")} +${extractFunction("sign_up_or_sign_in")} + +sign_up_or_sign_in +`; + const started = Date.now(); + const res = spawnSync("bash", ["-c", harness], { encoding: "utf8" }); + return { + code: res.status, + stdout: res.stdout ?? "", + stderr: res.stderr ?? "", + elapsedMs: Date.now() - started, + }; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test("succeeds when sign-up works on the first attempt", () => { + const r = runBootstrap([["200", "000"]]); + assert.equal(r.code, 0); + assert.match(r.stdout, /created admin user/); +}); + +test("falls back to sign-in for a data dir that already holds the admin", () => { + const r = runBootstrap([["409", "200"]]); + assert.equal(r.code, 0); + assert.match(r.stdout, /signed in existing admin user/); +}); + +test("retries past an auth stack that is not up yet", () => { + // The flake: health answers, the first sign-ups cannot connect, and the + // stack is serving moments later. One attempt failed here; the loop rides it out. + const r = runBootstrap([ + ["000", "000"], + ["000", "000"], + ["200", "000"], + ]); + assert.equal(r.code, 0); + assert.match(r.stdout, /created admin user/); +}); + +test("retries a 5xx and then signs in", () => { + const r = runBootstrap([ + ["000", "000"], + ["500", "500"], + ["409", "200"], + ]); + assert.equal(r.code, 0); + assert.match(r.stdout, /signed in existing admin user/); +}); + +test("gives up at the deadline rather than hanging", () => { + const r = runBootstrap([["000", "000"]], { retrySeconds: 4 }); + assert.equal(r.code, 1); + // Bounded on both sides: it must actually wait, and must not run away. + assert.ok(r.elapsedMs >= 4000, `gave up too early: ${r.elapsedMs}ms`); + assert.ok(r.elapsedMs < 30000, `overran the deadline: ${r.elapsedMs}ms`); +}); + +test("reports the HTTP status it saw, not just an empty body", () => { + // The reason this flake went a week without a diagnosis: the failure printed + // the response body, and a sign-up that never connected has none. The status + // was in hand the whole time and was not being surfaced. + const r = runBootstrap([["000", "000"]], { retrySeconds: 4 }); + assert.match(r.stderr, /Sign-up HTTP 000/); + assert.match(r.stderr, /Sign-in HTTP 000/); + assert.match(r.stderr, /attempts over 4s/); +}); diff --git a/scripts/docker-onboard-smoke.sh b/scripts/docker-onboard-smoke.sh index 548931b5574..dcb5573169f 100755 --- a/scripts/docker-onboard-smoke.sh +++ b/scripts/docker-onboard-smoke.sh @@ -160,9 +160,24 @@ get_with_cookies() { "$url" } -sign_up_or_sign_in() { - local signup_response="$TMP_DIR/signup.json" - local signup_status +# Seconds to keep retrying the admin bootstrap after /api/health first answers. +# `/api/health` returning 200 is not a promise that auth can serve a write: the +# container cold-installs paperclipai and brings up embedded postgres, and there +# is a window where health answers while a sign-up still fails. One un-retried +# attempt in that window is what made the nightly smoke flake — it failed on +# 2026-08-21, -23, -24, -26 and -27 while passing on the days in between, with +# identical inputs. +SMOKE_BOOTSTRAP_RETRY_SECONDS="${SMOKE_BOOTSTRAP_RETRY_SECONDS:-90}" + +# One sign-up, then one sign-in, reporting the status of each. Sign-in is the +# path for a container whose data dir already holds the admin — including one +# created by an earlier pass of the retry loop below, which is what makes the +# pair safe to repeat. +try_sign_up_or_sign_in() { + local signup_response="$1" + local signin_response="$2" + local signup_status signin_status + signup_status="$(post_json_with_cookies \ "$PAPERCLIP_PUBLIC_URL/api/auth/sign-up/email" \ "{\"name\":\"$SMOKE_ADMIN_NAME\",\"email\":\"$SMOKE_ADMIN_EMAIL\",\"password\":\"$SMOKE_ADMIN_PASSWORD\"}" \ @@ -172,8 +187,6 @@ sign_up_or_sign_in() { return 0 fi - local signin_response="$TMP_DIR/signin.json" - local signin_status signin_status="$(post_json_with_cookies \ "$PAPERCLIP_PUBLIC_URL/api/auth/sign-in/email" \ "{\"email\":\"$SMOKE_ADMIN_EMAIL\",\"password\":\"$SMOKE_ADMIN_PASSWORD\"}" \ @@ -183,11 +196,38 @@ sign_up_or_sign_in() { return 0 fi + LAST_SIGNUP_STATUS="$signup_status" + LAST_SIGNIN_STATUS="$signin_status" + return 1 +} + +sign_up_or_sign_in() { + local signup_response="$TMP_DIR/signup.json" + local signin_response="$TMP_DIR/signin.json" + local deadline=$((SECONDS + SMOKE_BOOTSTRAP_RETRY_SECONDS)) + local attempts=0 + + while :; do + attempts=$((attempts + 1)) + if try_sign_up_or_sign_in "$signup_response" "$signin_response"; then + return 0 + fi + if ((SECONDS >= deadline)); then + break + fi + sleep 2 + done + + # The statuses, not just the bodies. A failing sign-up here has historically + # returned an empty body — 000 when curl could not connect at all — so + # printing the body alone said nothing about what went wrong, which is why + # this flake went a week without a diagnosis. echo "Smoke bootstrap failed: could not sign up or sign in admin user" >&2 - echo "Sign-up response:" >&2 + echo "Gave up after $attempts attempts over ${SMOKE_BOOTSTRAP_RETRY_SECONDS}s." >&2 + echo "Sign-up HTTP ${LAST_SIGNUP_STATUS:-}, body:" >&2 cat "$signup_response" >&2 || true echo >&2 - echo "Sign-in response:" >&2 + echo "Sign-in HTTP ${LAST_SIGNIN_STATUS:-}, body:" >&2 cat "$signin_response" >&2 || true echo >&2 return 1