From 49a2b7b245fec7b78cb4467cb547c06be6d30ea1 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:22:57 -0600 Subject: [PATCH] test(recovery): record a live process's OS-observed start time in owner-proof tests (#21847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Recovery continues a cut-off task automatically. It first lets a previous process that still runs on this host settle, for about ten minutes at most. > - A server test proves that wait. It failed once on master at 03309f1650 in the "Cloud readiness" run, and the failure blocked the staging release. > - The test stamps the run with `Date.now() - process.uptime()`. The product compares the stamp with the start time that the OS reports, and a gap of 1 s or more means "a different process with a reused PID". > - The OS time is truncated to whole seconds (Linux `btime`, macOS `ps -o lstart`), and Node starts some milliseconds after the fork. Thus the gap is sometimes 1 s or more, and the product correctly continues the task. > - This pull request makes the test stamp the same OS-observed start time that production records. > - The benefit is a deterministic test that still proves the wait and its ten-minute upper bound. ## Linked Issues or Issue Description **What happened?** `server/src/services/fresh-continuation-fallback.test.ts > lets a previous process still running on this host settle first, for about ten minutes at most` failed on master (Cloud readiness run 43776843790, job "verify / General tests (server (2/10))"). Received `{ kind: "continued" }`, expected `{ kind: "exact_recovery_pending", reason: "previous_process_running" }`. **Expected behavior** The test passes on every host. It only fails when the product does not wait for a live previous process. **Steps to reproduce** 1. Run the test about 15–30 times on macOS under CPU load (for example 24 busy loops on 18 cores). 2. About 1 in 13 runs fails with the same assertion as CI. 3. On Linux, the gap is `frac(boot time)` plus Node startup. Thus a runner whose boot time has a fractional second near 1 fails, and other runners pass. **Paperclip version or commit** 03309f1650 (master). **Additional context** This is not a product regression. Production writes `processStartedAt` from `readProcessStartedAt` (`persistHeartbeatRunProcessMetadata`), which is the same source that the owner proof reads. Thus production compares two equal values. Neither #21831 nor #21835 changed this path. ## What Changed - `fresh-continuation-fallback.test.ts`: `liveIdentity()` now records `observedProcessStartedAt(process.pid)`, the OS-observed start time that a run records. It throws if the host cannot report the time, so the test cannot pass by accident. - `completed-turn-cleanup-hold-settle.test.ts`: the live controller stamp now uses `currentProcessStartedAtMs()`, the value that this server stamps on its own claims. It had the same `Date.now() - process.uptime()` pattern. - No product code changed. No assertion changed or became weaker. The ten-minute upper bound assertion stays. ## Verification - Gap probe (fresh Node processes, `Date.now() - uptime` minus the OS start time): - macOS under load: 2 of 200 processes had a gap of 1 s or more (max 1017 ms). - Linux (`node:24-bookworm-slim`): gap 340–396 ms, of which 337–345 ms is `btime` truncation. This value is fixed for each host boot, so some CI hosts are always near or past 1 s. - `npx vitest run src/services/fresh-continuation-fallback.test.ts -t "previous process still running|recycled PID"`, repeated under CPU load: see the before/after counts in the comment below. - `npx vitest run src/__tests__/completed-turn-cleanup-hold-settle.test.ts`: 10 passed. ## Risks - Low risk. Only two test files changed. - If a CI host cannot read a process start time, the changed tests now throw a clear error. Before this change the product would also return "unprovable" there. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use (shell, file edits) and extended reasoning. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../completed-turn-cleanup-hold-settle.test.ts | 10 ++++++++-- .../services/fresh-continuation-fallback.test.ts | 13 ++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/server/src/__tests__/completed-turn-cleanup-hold-settle.test.ts b/server/src/__tests__/completed-turn-cleanup-hold-settle.test.ts index 1f0d1a86b32..dd5bb9e7936 100644 --- a/server/src/__tests__/completed-turn-cleanup-hold-settle.test.ts +++ b/server/src/__tests__/completed-turn-cleanup-hold-settle.test.ts @@ -18,7 +18,7 @@ import { workspaceOperations, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -import { currentControllerBootId, currentProcessSpaceHash } from "../services/controller-process-identity.js"; +import { currentControllerBootId, currentProcessSpaceHash, currentProcessStartedAtMs } from "../services/controller-process-identity.js"; import { getExecutionBlocker } from "../services/execution-blocker.js"; import { heartbeatService } from "../services/heartbeat.js"; import { deliverReconciledExecutions } from "../services/execution-recovery-resolution.js"; @@ -36,6 +36,12 @@ async function exitedPid(): Promise { return pid; } +async function ownProcessStartedAt(): Promise { + const startedAt = await currentProcessStartedAtMs(); + if (startedAt === null) throw new Error("This host cannot report this process's start time"); + return new Date(startedAt); +} + /** * The staging shape of a run that completed its provider turn after a restart * orphaned one tool callback. Close could not prove the drain, so the run @@ -104,7 +110,7 @@ suite("completed-turn cleanup holds on the shared ownership path", () => { leaseOwner: null, failureCode: "native_session_cleanup_quarantined", recoveryState: "blocked", controllerGeneration: 2, ...(options.controllerAlive ? { controllerBootId: currentControllerBootId(), controllerPid: process.pid, controllerHostHash: currentProcessSpaceHash(), - controllerProcessStartedAt: new Date(Date.now() - process.uptime() * 1_000) } + controllerProcessStartedAt: await ownProcessStartedAt() } : { controllerBootId: options.controllerBootId ?? randomUUID(), controllerPid: await exitedPid(), controllerHostHash: options.hostHashUnrecorded ? null : currentProcessSpaceHash(), controllerProcessStartedAt: new Date(finishedAt.getTime() - 20 * 60_000) }) }); diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 0fb268e48ae..33a1dab5320 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -16,7 +16,7 @@ import { } from "./fresh-continuation-fallback.js"; import { heartbeatService } from "./heartbeat.js"; import { buildExecutionContinuation } from "./execution-continuation.js"; -import { currentProcessSpaceHash } from "./controller-process-identity.js"; +import { currentProcessSpaceHash, observedProcessStartedAt } from "./controller-process-identity.js"; import { executionControlSchedule } from "./execution-control-work.js"; import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js"; import { taskInterruptionPresentation } from "./task-interruption-presentation.js"; @@ -748,8 +748,11 @@ suite("fresh-continuation fallback", () => { }, 60_000); // The recorded process identity (host, PID space, start time) of this process. - const liveIdentity = () => ({ processPid: process.pid, processHostHash: currentProcessSpaceHash(), - processStartedAt: new Date(Date.now() - process.uptime() * 1_000) }); + const liveIdentity = async () => { + const startedAt = await observedProcessStartedAt(process.pid); + if (startedAt === null) throw new Error("This host cannot report this process's start time"); + return { processPid: process.pid, processHostHash: currentProcessSpaceHash(), processStartedAt: new Date(startedAt) }; + }; /** Moves the recorded start of the pending wait into the past. */ async function waitedFor(actionId: string, ms: number) { @@ -759,7 +762,7 @@ suite("fresh-continuation fallback", () => { it("lets a previous process still running on this host settle first, for about ten minutes at most", async () => { const f = await unfinishedRun({ runtime: "legacy", errorCode: "process_lost" }); - await db.update(heartbeatRuns).set(liveIdentity()).where(eq(heartbeatRuns.id, f.runId)); + await db.update(heartbeatRuns).set(await liveIdentity()).where(eq(heartbeatRuns.id, f.runId)); const first = await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" }); expect(first).toMatchObject({ kind: "exact_recovery_pending", reason: "previous_process_running" }); // Rechecks back off from one minute. @@ -792,7 +795,7 @@ suite("fresh-continuation fallback", () => { it("does not mistake a recycled PID for the old writer", async () => { const f = await unfinishedRun({ runtime: "legacy", errorCode: "process_lost" }); // The PID is alive now, but it belongs to a process that started after the old one was recorded. - await db.update(heartbeatRuns).set({ ...liveIdentity(), processStartedAt: new Date("2020-01-01T00:00:00Z") }) + await db.update(heartbeatRuns).set({ ...(await liveIdentity()), processStartedAt: new Date("2020-01-01T00:00:00Z") }) .where(eq(heartbeatRuns.id, f.runId)); expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })) .toMatchObject({ kind: "continued" });