From 997ea5fd17d5ec6fcb03bf015f8f448270db8475 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:47:22 -0500 Subject: [PATCH] test(server): retire duplicate process-loss recovery execution (#16088) Co-Authored-By: Paperclip --- ...audit-r16-heartbeat-recovery-controls.json | 97 +++++++++++++++++++ ...10-12-test-audit-r16-heartbeat-recovery.md | 84 ++++++++++++++++ .../heartbeat-process-recovery.test.ts | 36 +------ 3 files changed, 186 insertions(+), 31 deletions(-) create mode 100644 doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery-controls.json create mode 100644 doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery.md diff --git a/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery-controls.json b/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery-controls.json new file mode 100644 index 00000000000..6179db6be79 --- /dev/null +++ b/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery-controls.json @@ -0,0 +1,97 @@ +{ + "parentPR": 16086, + "parentBranch": "codex/test-audit-r15-environment-runtime", + "parentHead": "c0414ac02e625a1e635857895b74581c37201500", + "result": "PROVEN", + "registrationsBefore": 399, + "registrationsAfter": 398, + "declarationsBefore": 249, + "declarationsAfter": 248, + "retiredCase": "heartbeat orphaned process recovery re-enqueues continuation for stranded in-progress work with no active run", + "keeper": "heartbeat orphaned process recovery re-enqueues stranded in-progress work after a failed/process_lost run before escalating", + "deletedTestLOC": 31, + "movedAssertionLOC": 5, + "netTestLOCDelta": -26, + "candidateTestSHA256": "645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6", + "parentSourceHashes": { + "server/src/__tests__/heartbeat-process-recovery.test.ts": "bf7c888eaba44a56d12823bb948026d5db44f2249a75a6cf083343edbd7c0673", + "server/src/services/heartbeat/retries.ts": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a", + "server/src/services/heartbeat.ts": "9ac217e97e5319c850924f083df32be43df11f33f8970ff8efb4a5f776a0c1d9", + "server/src/services/recovery/service.ts": "c414df876e7df64e701a752513db8e5cf7365589294d50da0e1396788a9207db" + }, + "sourceRestoredBeforeBranchCreation": true, + "controls": [ + { + "phase": "baseline", + "exitCode": 0, + "tests": 399, + "passed": 399, + "failed": 0, + "pending": 0, + "failures": [], + "ownerSHA256": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a", + "testSHA256": "bf7c888eaba44a56d12823bb948026d5db44f2249a75a6cf083343edbd7c0673" + }, + { + "phase": "candidate", + "exitCode": 0, + "tests": 398, + "passed": 398, + "failed": 0, + "pending": 0, + "failures": [], + "ownerSHA256": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a", + "testSHA256": "645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6" + }, + { + "phase": "revert", + "exitCode": 0, + "tests": 399, + "passed": 399, + "failed": 0, + "pending": 0, + "failures": [], + "ownerSHA256": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a", + "testSHA256": "bf7c888eaba44a56d12823bb948026d5db44f2249a75a6cf083343edbd7c0673" + }, + { + "phase": "mutation-keeper", + "exitCode": 1, + "tests": 398, + "passed": 0, + "failed": 1, + "pending": 397, + "failures": [ + { + "name": "heartbeat orphaned process recovery re-enqueues stranded in-progress work after a failed/process_lost run before escalating", + "messages": [ + "AssertionError: expected { \u2026(78) } to match object { status: 'scheduled_retry', \u2026(1) }\n(84 matching properties omitted from actual)\n at Proxy. (file:///node_modules/.pnpm/vitest@5.0.3_@opentelemetry+api@1.9.1_@types+node@24.13.3_jsdom@30.0.1_@noble+hashes@2.4.0__v_wr5mxfuu6kgge6st623q4v5egm/node_modules/vitest/dist/chunks/index.C4tKZ0yW.js:2050:10)\n at Proxy. (file:///node_modules/.pnpm/vitest@5.0.3_@opentelemetry+api@1.9.1_@types+node@24.13.3_jsdom@30.0.1_@noble+hashes@2.4.0__v_wr5mxfuu6kgge6st623q4v5egm/node_modules/vitest/dist/chunks/index.C4tKZ0yW.js:1944:14)\n at Proxy.methodWrapper (file:///node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n at server/src/__tests__/heartbeat-process-recovery.test.ts:10386:24\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file:///node_modules/.pnpm/vitest@5.0.3_@opentelemetry+api@1.9.1_@types+node@24.13.3_jsdom@30.0.1_@noble+hashes@2.4.0__v_wr5mxfuu6kgge6st623q4v5egm/node_modules/vitest/dist/chunks/run.BTlFXlnv.js:2783:20" + ] + } + ], + "ownerSHA256": "cff836de7a98e47e845ad7f7393e4c98fcfea3fecb0ba5bdb9fcc92fc9673c84", + "testSHA256": "645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6" + }, + { + "phase": "restoration", + "exitCode": 0, + "tests": 398, + "passed": 398, + "failed": 0, + "pending": 0, + "failures": [], + "ownerSHA256": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a", + "testSHA256": "645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6" + } + ], + "temporaryMutation": { + "owner": "server/src/services/heartbeat/retries.ts", + "line": 1049, + "before": "scheduledRetryAttempt: schedule.attempt", + "after": "scheduledRetryAttempt: schedule.attempt + 1", + "keeperAssertionLine": 10386, + "selection": "Only exact failed/process_lost keeper selected; all other cases are unselected.", + "expectedAttempt": 1, + "observedAttempt": 2 + } +} diff --git a/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery.md b/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery.md new file mode 100644 index 00000000000..5174228c42f --- /dev/null +++ b/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery.md @@ -0,0 +1,84 @@ +# Heartbeat recovery test audit — 2026-10-12 + +Parent: #16086 (`codex/test-audit-r15-environment-runtime`), exact head +`c0414ac02e625a1e635857895b74581c37201500`. + +## Change and keeper + +The standalone `re-enqueues continuation for stranded in-progress work with no active run` +case at `heartbeat-process-recovery.test.ts:11192–11221` repeats the existing +`re-enqueues stranded in-progress work after a failed/process_lost run before escalating` +table row at 10360. + +Both call `seedStrandedIssueFixture` with `status: in_progress` and +`runStatus: failed`. The standalone test omits `runErrorCode`; the helper at 1132–1136 +persists `process_lost` as its default. The table supplies that same value. +Every other supplied argument, default, and owner action matches. Generated +record IDs vary between independent fixture runs. + +Move five unique assertion lines into the keeper: a truthy retry ID and +`{ status: scheduled_retry, scheduledRetryAttempt: 1 }`. Existing keeper checks +retain zero dispatch, one continuation, no escalation, exact issue IDs, two +agent runs, `transient_failure`, and absent `modelProfile`. The keeper also +checks source task/run binding, no recovery children, and eventual settlement. +The failed/adapter_failed and timed_out/adapter_timed_out rows remain and gain +the durable retry assertions. + +One test file: +5/−31 lines, net 26 fewer test lines. One registered execution +is retired (399→398). This is execution retirement, with no production change. +Crash/restart, ownership, cancellation, persistence, retry-budget, and failure +isolation contracts remain. No CI time saving is claimed from local samples. + +## Exact-parent sequential proof + +| Phase | Passed | Failed | Filtered/pending | +|---|---|---|---| +| Full baseline | 399 | 0 | 0 | +| Full candidate | 398 | 0 | 0 | +| Full revert | 399 | 0 | 0 | +| Temporary production mutation, exact keeper | 0 | 1 | 397 | +| Owner restored, full candidate | 398 | 0 | 0 | + +The mutation changes the real scheduler write at `heartbeat/retries.ts:1049` +from `schedule.attempt` to `schedule.attempt + 1`. The exact keeper fails at +the moved retry-state assertion, candidate line 10386, expecting attempt 1 and receiving 2. Only that keeper is +selected; other cases are filtered. The persisted attempt feeds durable +failure-retry accounting in `execution-recovery-attempt.ts:40–49`. +The recovery/scheduler owner is not mocked. The mutation is absent from this PR. +After restoring the owner, all 398 candidate cases pass. Both source files were +then restored byte-for-byte and the checkout was clean before branch creation. +The final patch is the exact candidate bytes that passed restoration. + +Runtime-title comparison removes only the standalone registration and adds +no cases. The original full-file ledger contains 249 declarations / 399 runtime +cases; the candidate has 248 / 398. All declarations were reviewed. Near-overlaps +with different queued wakes, timer claims, continuation ancestry, or crash points +were retained. + +```sh +PAPERCLIP_HOME=/private/tmp/r16-hbr/h \ +PAPERCLIP_CONFIG=/private/tmp/r16-hbr/h/config.json \ +PAPERCLIP_INSTANCE_ID=heartbeat-recovery-audit TMPDIR=/private/tmp/r16-hbr/t NODE_ENV=test \ +node node_modules/vitest/vitest.mjs run --project @paperclipai/server \ +server/src/__tests__/heartbeat-process-recovery.test.ts --exclude '**/dist/**' +``` + +Full repository typecheck/build/test preflight is delegated to isolated CI, +as in the parent audit. The local worktree uses shared external dependency +symlinks, and preflight rewrites workspace links. Every CI gate must pass +before a green handoff. The full owner uses the checked-in server configuration +and its real embedded Postgres. + +## Hashes and receipts + +- Parent test SHA-256: `bf7c888eaba44a56d12823bb948026d5db44f2249a75a6cf083343edbd7c0673`. +- Candidate test SHA-256: `645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6`. +- Restored production retry owner SHA-256: `9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a`. + +Machine receipts are in `2026-10-12-test-audit-r16-heartbeat-recovery-controls.json`. +Full JSON/log output, patch, exact keeper map, and whole-file ledger are in +`/private/tmp/r16-heartbeat-recovery/`. The earlier read-only discovery is in +`/private/tmp/heartbeat-recovery-full-audit/`. This parent replay uses fresh +sequential controls; it does not reuse the earlier parent proof. + +Model: OpenAI `gpt-6.1-sol`; the host does not expose its context-window size. diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 66f16134ef1..d719c36421f 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -10648,6 +10648,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(runs).toHaveLength(2); const retryRun = runs.find((row) => row.id !== runId); + expect(retryRun?.id).toBeTruthy(); + expect(retryRun).toMatchObject({ + status: "scheduled_retry", + scheduledRetryAttempt: 1, + }); expect( retryRun?.contextSnapshot as Record | undefined, ).toMatchObject({ @@ -11455,37 +11460,6 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } }); - it("re-enqueues continuation for stranded in-progress work with no active run", async () => { - const { companyId, agentId, issueId, runId } = - await seedStrandedIssueFixture({ - status: "in_progress", - runStatus: "failed", - }); - const heartbeat = heartbeatService(db); - - const result = await heartbeat.reconcileStrandedAssignedIssues(); - expect(result.dispatchRequeued).toBe(0); - expect(result.continuationRequeued).toBe(1); - expect(result.escalated).toBe(0); - expect(result.issueIds).toEqual([issueId]); - - const runs = await selectRecoveryRunsByAgent(agentId); - expect(runs).toHaveLength(2); - - const retryRun = runs.find((row) => row.id !== runId); - expect(retryRun?.id).toBeTruthy(); - expect( - (retryRun?.contextSnapshot as Record)?.retryReason, - ).toBe("transient_failure"); - expect( - retryRun?.contextSnapshot as Record, - ).not.toHaveProperty("modelProfile"); - expect(retryRun).toMatchObject({ - status: "scheduled_retry", - scheduledRetryAttempt: 1, - }); - }); - it("does not run generic continuation recovery for a paused unfinished session goal", async () => { const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({