diff --git a/scripts/__tests__/docker-onboard-smoke-bootstrap.test.mjs b/scripts/__tests__/docker-onboard-smoke-bootstrap.test.mjs new file mode 100644 index 00000000000..b0bf2e97577 --- /dev/null +++ b/scripts/__tests__/docker-onboard-smoke-bootstrap.test.mjs @@ -0,0 +1,151 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const script = new URL("../docker-onboard-smoke.sh", import.meta.url).pathname; + +/** + * The admin bootstrap in `docker-onboard-smoke.sh`, exercised without Docker. + * + * `/api/health` answering 200 is not a promise that auth can serve a write: the + * container cold-installs paperclipai and brings up embedded postgres, and there + * is a window where health answers while a sign-up still fails. A single + * un-retried attempt in that window is what made the nightly release smoke flake + * — it failed on 2026-08-21, -23, -24, -26 and -27 while passing on the days + * between, on identical inputs. + * + * These tests drive the real functions out of the shipped script with a stubbed + * request helper. Extracting them rather than restating them keeps the test + * honest: a copy would drift from what actually runs in CI. + */ + +const source = fs.readFileSync(script, "utf8"); + +function extractFunction(name) { + // Anchored to the start of a line: a bare indexOf for `sign_up_or_sign_in() {` + // also matches inside `try_sign_up_or_sign_in() {`, which is defined first. + const marker = `\n${name}() {`; + const at = source.indexOf(marker); + assert.notEqual(at, -1, `${name} not found at column 0 in docker-onboard-smoke.sh`); + const start = at + 1; + const end = source.indexOf("\n}\n", start); + assert.notEqual(end, -1, `${name} has no closing brace at column 0`); + return source.slice(start, end + 3); +} + +/** + * Runs the bootstrap against a scripted sequence of responses. + * + * Each entry in `attempts` is `[signUpStatus, signInStatus]` for one pass of the + * loop; "000" stands for curl failing to connect at all, which is what the + * server returns while it is still coming up — and which produces the empty + * body the real failures showed. + */ +function runBootstrap(attempts, { retrySeconds = 6 } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "smoke-bootstrap-")); + try { + const table = attempts.map((pair) => pair.join(" ")).join("\n"); + fs.writeFileSync(path.join(dir, "responses"), `${table}\n`); + + // The helper is called as `$(...)`, so a counter held in a shell variable + // would die with that subshell. It lives in a file for the same reason. + const harness = ` +set -uo pipefail +TMP_DIR=${JSON.stringify(dir)} +COUNTER="$TMP_DIR/attempt" +echo 0 > "$COUNTER" +PAPERCLIP_PUBLIC_URL="http://stub" +SMOKE_ADMIN_NAME="Smoke Admin" +SMOKE_ADMIN_EMAIL="smoke-admin@paperclip.local" +SMOKE_ADMIN_PASSWORD="pw" +SMOKE_BOOTSTRAP_RETRY_SECONDS=${retrySeconds} + +post_json_with_cookies() { + local url="$1" body="$2" out="$3" + local idx line code + idx="$(cat "$COUNTER")" + line="$(sed -n "$((idx + 1))p" "$TMP_DIR/responses")" + [[ -z "$line" ]] && line="000 000" + if [[ "$url" == *sign-up* ]]; then + code="\${line%% *}" + [[ "$code" =~ ^2 ]] && echo $((idx + 1)) > "$COUNTER" + else + code="\${line##* }" + echo $((idx + 1)) > "$COUNTER" + fi + if [[ "$code" == "000" ]]; then : > "$out"; else echo "{\\"stub\\":$code}" > "$out"; fi + printf '%s' "$code" +} + +${extractFunction("try_sign_up_or_sign_in")} +${extractFunction("sign_up_or_sign_in")} + +sign_up_or_sign_in +`; + const started = Date.now(); + const res = spawnSync("bash", ["-c", harness], { encoding: "utf8" }); + return { + code: res.status, + stdout: res.stdout ?? "", + stderr: res.stderr ?? "", + elapsedMs: Date.now() - started, + }; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test("succeeds when sign-up works on the first attempt", () => { + const r = runBootstrap([["200", "000"]]); + assert.equal(r.code, 0); + assert.match(r.stdout, /created admin user/); +}); + +test("falls back to sign-in for a data dir that already holds the admin", () => { + const r = runBootstrap([["409", "200"]]); + assert.equal(r.code, 0); + assert.match(r.stdout, /signed in existing admin user/); +}); + +test("retries past an auth stack that is not up yet", () => { + // The flake: health answers, the first sign-ups cannot connect, and the + // stack is serving moments later. One attempt failed here; the loop rides it out. + const r = runBootstrap([ + ["000", "000"], + ["000", "000"], + ["200", "000"], + ]); + assert.equal(r.code, 0); + assert.match(r.stdout, /created admin user/); +}); + +test("retries a 5xx and then signs in", () => { + const r = runBootstrap([ + ["000", "000"], + ["500", "500"], + ["409", "200"], + ]); + assert.equal(r.code, 0); + assert.match(r.stdout, /signed in existing admin user/); +}); + +test("gives up at the deadline rather than hanging", () => { + const r = runBootstrap([["000", "000"]], { retrySeconds: 4 }); + assert.equal(r.code, 1); + // Bounded on both sides: it must actually wait, and must not run away. + assert.ok(r.elapsedMs >= 4000, `gave up too early: ${r.elapsedMs}ms`); + assert.ok(r.elapsedMs < 30000, `overran the deadline: ${r.elapsedMs}ms`); +}); + +test("reports the HTTP status it saw, not just an empty body", () => { + // The reason this flake went a week without a diagnosis: the failure printed + // the response body, and a sign-up that never connected has none. The status + // was in hand the whole time and was not being surfaced. + const r = runBootstrap([["000", "000"]], { retrySeconds: 4 }); + assert.match(r.stderr, /Sign-up HTTP 000/); + assert.match(r.stderr, /Sign-in HTTP 000/); + assert.match(r.stderr, /attempts over 4s/); +}); diff --git a/scripts/docker-onboard-smoke.sh b/scripts/docker-onboard-smoke.sh index 548931b5574..dcb5573169f 100755 --- a/scripts/docker-onboard-smoke.sh +++ b/scripts/docker-onboard-smoke.sh @@ -160,9 +160,24 @@ get_with_cookies() { "$url" } -sign_up_or_sign_in() { - local signup_response="$TMP_DIR/signup.json" - local signup_status +# Seconds to keep retrying the admin bootstrap after /api/health first answers. +# `/api/health` returning 200 is not a promise that auth can serve a write: the +# container cold-installs paperclipai and brings up embedded postgres, and there +# is a window where health answers while a sign-up still fails. One un-retried +# attempt in that window is what made the nightly smoke flake — it failed on +# 2026-08-21, -23, -24, -26 and -27 while passing on the days in between, with +# identical inputs. +SMOKE_BOOTSTRAP_RETRY_SECONDS="${SMOKE_BOOTSTRAP_RETRY_SECONDS:-90}" + +# One sign-up, then one sign-in, reporting the status of each. Sign-in is the +# path for a container whose data dir already holds the admin — including one +# created by an earlier pass of the retry loop below, which is what makes the +# pair safe to repeat. +try_sign_up_or_sign_in() { + local signup_response="$1" + local signin_response="$2" + local signup_status signin_status + signup_status="$(post_json_with_cookies \ "$PAPERCLIP_PUBLIC_URL/api/auth/sign-up/email" \ "{\"name\":\"$SMOKE_ADMIN_NAME\",\"email\":\"$SMOKE_ADMIN_EMAIL\",\"password\":\"$SMOKE_ADMIN_PASSWORD\"}" \ @@ -172,8 +187,6 @@ sign_up_or_sign_in() { return 0 fi - local signin_response="$TMP_DIR/signin.json" - local signin_status signin_status="$(post_json_with_cookies \ "$PAPERCLIP_PUBLIC_URL/api/auth/sign-in/email" \ "{\"email\":\"$SMOKE_ADMIN_EMAIL\",\"password\":\"$SMOKE_ADMIN_PASSWORD\"}" \ @@ -183,11 +196,38 @@ sign_up_or_sign_in() { return 0 fi + LAST_SIGNUP_STATUS="$signup_status" + LAST_SIGNIN_STATUS="$signin_status" + return 1 +} + +sign_up_or_sign_in() { + local signup_response="$TMP_DIR/signup.json" + local signin_response="$TMP_DIR/signin.json" + local deadline=$((SECONDS + SMOKE_BOOTSTRAP_RETRY_SECONDS)) + local attempts=0 + + while :; do + attempts=$((attempts + 1)) + if try_sign_up_or_sign_in "$signup_response" "$signin_response"; then + return 0 + fi + if ((SECONDS >= deadline)); then + break + fi + sleep 2 + done + + # The statuses, not just the bodies. A failing sign-up here has historically + # returned an empty body — 000 when curl could not connect at all — so + # printing the body alone said nothing about what went wrong, which is why + # this flake went a week without a diagnosis. echo "Smoke bootstrap failed: could not sign up or sign in admin user" >&2 - echo "Sign-up response:" >&2 + echo "Gave up after $attempts attempts over ${SMOKE_BOOTSTRAP_RETRY_SECONDS}s." >&2 + echo "Sign-up HTTP ${LAST_SIGNUP_STATUS:-}, body:" >&2 cat "$signup_response" >&2 || true echo >&2 - echo "Sign-in response:" >&2 + echo "Sign-in HTTP ${LAST_SIGNIN_STATUS:-}, body:" >&2 cat "$signin_response" >&2 || true echo >&2 return 1