diff --git a/doc/plans/2026-10-31-zero-downtime-web-deploys.md b/doc/plans/2026-10-31-zero-downtime-web-deploys.md index 842a6e5854d..fac22b56194 100644 --- a/doc/plans/2026-10-31-zero-downtime-web-deploys.md +++ b/doc/plans/2026-10-31-zero-downtime-web-deploys.md @@ -1,5 +1,54 @@ # Zero-downtime web deployments +Current state (2026-11-02 18:31 UTC): cleanup recovery PR +[#18633](https://github.com/paperclipai/paperclip/pull/18633) merged as +`60a99307b3bb1062078d35af6677e3f377d8bab8`. Its final head passed all PR CI, +full local typecheck and build, and seven focused cleanup regressions. Greptile +reviewed that head at 5/5; both review threads are resolved. The duplicate full +local test run was stopped after six chat shards passed, while general suites +were still running. Its incomplete local result is preserved separately from +successful full PR CI. + +Canonical Linux release verification failed on the unchanged two-second busy-row +startup test on several consecutive revisions, including the cleanup merge. +The held-row diagnostic completed in 420 ms on macOS without a database lock +wait. Inspection found that `currentNativeControllerIdentity` always called the +server-info snapshot, which synchronously probes Git three times even when the +OS process birth time is available. Controlled one-second delays in real Git +commands reproduced the exact startup assertion failure with real PostgreSQL. +The follow-up separates the constant fallback process birth time from diagnostic +Git metadata. New regressions cover both available and unavailable OS birth time. +The two-second startup limit and release gates remain unchanged. Linux release +verification must pass before the new source can enter hosted qualification. + +The follow-up is [#18649](https://github.com/paperclipai/paperclip/pull/18649). +Its first head received two 5/5 reviews and passed every CI test shard except +an existing voice-retention precision test. That test advances its fake clock +automatically while asserting behavior across a one-millisecond boundary. A +controlled 60 ms database-entry delay reproduced the exact failure. Holding the +clock fixed makes the same assertion pass with that delay; the final correction +only disables automatic clock advancement in that one precision test. The +original CI failure and both controlled results are retained. No production +voice code or assertion was weakened. + +The earlier ten-roll result remains evidence for its recorded images. Neither +the cleanup fix nor the controller-identity follow-up has reached the staging +fixture. Final acceptance still requires a qualified compatible image pair, +fresh staging replacements, natural idle and browser sleep/wake, followed by the +isolated production canary. Cloud source is `b44304603d`; its private packages +require comparison against the new Core SDK and shared archive. Earlier candidate +Core `42c64d4ec4` differs in SDK; `b9e94b85a6` differs in shared types. Neither is +a valid rollback pair for the cleanup merge. No release gate was bypassed. + +The production browser was signed in as Dotta B on the ordinary new-organization +form. No organization was submitted and no production config changed. The only +hosted target changed so far is the pinned staging qualification fixture. +Disposable local PostgreSQL and Moto S3 fixtures passed an independent object +read/write preflight. Docker subsequently stopped; local container qualification +requires restoring that dependency. Hosted acceptance uses real object storage. + +Earlier retained evidence and failure history follows. + Date: 2026-10-31. Status (2026-11-02 17:23 UTC): Core #18554 is merged as `1a6017d6460267ad89d7e1784e2b521c28d8e9ac`. The ten-roll and native-agent proof used Cloud `461c4aea28`. Separately reviewed Cloud #1091/#1092 subsequently diff --git a/server/src/__tests__/voice-inbound-retention.test.ts b/server/src/__tests__/voice-inbound-retention.test.ts index 6bdedd256d5..6925c795d39 100644 --- a/server/src/__tests__/voice-inbound-retention.test.ts +++ b/server/src/__tests__/voice-inbound-retention.test.ts @@ -145,6 +145,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(await row(call.id)).toBeUndefined(); expect(coordinator!.nextWakeAt()).toBeNull(); await quiet(); }); it("rounds PostgreSQL submillisecond deadlines up and never deletes early", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false }); const call = await admission(); const dueAt = Math.trunc(Date.now()) + 60_000; await db.execute(sql`UPDATE chat_voice_inbound_calls SET updated_at = to_timestamp(${dueAt - week}::double precision / 1000) + interval '0.0005 seconds' WHERE id = ${call.id}`); const service = voiceInboundRetention(db), [entry] = await service.load({ kind: "admission", id: call.id }); diff --git a/server/src/server-info.ts b/server/src/server-info.ts index 455e622c870..6d3e861e6bb 100644 --- a/server/src/server-info.ts +++ b/server/src/server-info.ts @@ -172,6 +172,11 @@ const GIT_INFO_CACHE_TTL_MS = 3000; const processStartedAt = new Date().toISOString(); let gitInfoCache: { value: ServerGitInfo; expiresAt: number } | null = null; +// Lifecycle identity must not run the synchronous Git probes used by diagnostics. +export function getServerProcessStartedAt(): string { + return processStartedAt; +} + export function getServerInfoSnapshot( opts: { now?: number; diff --git a/server/src/services/native-runtime/native-controller-identity.test.ts b/server/src/services/native-runtime/native-controller-identity.test.ts new file mode 100644 index 00000000000..fd11591da9a --- /dev/null +++ b/server/src/services/native-runtime/native-controller-identity.test.ts @@ -0,0 +1,37 @@ +import { afterEach, expect, it, vi } from "vitest"; +import * as childProcess from "node:child_process"; +import * as hotRestart from "../hot-restart.js"; +import { getServerInfoSnapshot, resetServerInfoCacheForTests } from "../../server-info.js"; +import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; + +vi.mock("node:child_process", async importOriginal => ({ + ...await importOriginal(), + execFileSync: vi.fn(() => { + throw new Error("Git metadata is unavailable; identity must still work"); + }), +})); + +afterEach(() => { + vi.restoreAllMocks(); + resetServerInfoCacheForTests(); +}); + +it.each([true, false])("keeps controller identity independent of Git commands (OS start available=%s)", async available => { + const observed = "2026-11-02T12:00:00.000Z"; + const fallback = getServerInfoSnapshot({ gitCommand: () => { throw new Error("no checkout"); } }).processStartedAt; + resetServerInfoCacheForTests(); + vi.spyOn(hotRestart, "readProcessStartedAt").mockImplementation(async () => { + if (!available) throw new Error("process start unavailable"); + return observed; + }); + const subprocess = vi.mocked(childProcess.execFileSync); + subprocess.mockClear(); + + const first = await currentNativeControllerIdentity(); + const second = await currentNativeControllerIdentity(); + + expect(first).toEqual(second); + expect(first.pid).toBe(process.pid); + expect(first.processStartedAt.toISOString()).toBe(available ? observed : fallback); + expect(subprocess).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 96a4112e5a6..a27af96ac3b 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -31,7 +31,7 @@ import { } from "@paperclipai/db"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; import { readProcessStartedAt } from "../hot-restart.js"; -import { getServerInfoSnapshot } from "../../server-info.js"; +import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js"; @@ -157,7 +157,7 @@ async function observedProcessStart(pid: number): Promise { export async function currentNativeControllerIdentity(): Promise { const observed = await observedProcessStart(process.pid); - const fallback = new Date(getServerInfoSnapshot().processStartedAt); + const fallback = new Date(getServerProcessStartedAt()); return { bootId: controllerBootId, pid: process.pid,