From f6f42a8df0e7974a9ce490209a3ff7cab511f9c8 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 11 Oct 2026 09:12:35 -0500 Subject: [PATCH] Record reviewed idle fixes and integrated build proof Keep full package live qualification separate from the passing control and regression checks. Co-Authored-By: Paperclip --- doc/plans/2026-10-10-boat-validation.md | 12 +- .../evidence/idle-fixes-verification.json | 179 ++++++++++++++++++ .../evidence/stack-e077-typecheck-build.json | 32 ++++ 3 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 doc/plans/assets/2026-10-10-boat/evidence/idle-fixes-verification.json create mode 100644 doc/plans/assets/2026-10-10-boat/evidence/stack-e077-typecheck-build.json diff --git a/doc/plans/2026-10-10-boat-validation.md b/doc/plans/2026-10-10-boat-validation.md index c31ffc82980..9b14dc0eb3f 100644 --- a/doc/plans/2026-10-10-boat-validation.md +++ b/doc/plans/2026-10-10-boat-validation.md @@ -4,21 +4,21 @@ Historical screenshots and receipts are preserved on the [fixed evidence branch](https://github.com/paperclipai/paperclip/tree/5909bfac9fa41405beb3fe8a65253dc28ba52835/doc/plans/assets/2026-10-10-boat). Historical links below use that exact commit; the implementation PR keeps a bounded set of current proofs. -## Current source boundary (2026-10-11 14:01 UTC) +## Current source boundary (2026-10-11 14:12 UTC) **The live independent-file control passed; full Claude package acceptance is still pending.** The four-file experiment ran on unchanged staging29 source `05f3725af838b5a074707df1d6ed1e193bdfe49e`. After normal warm timeout and completed archive, the same saved Codex task read each file once through the unchanged bounded-read guard. The hardlinked control reproduced **nlink 1→2 / LINK_COUNT rejection**. Both independent files passed with unchanged hashes and nlink 1. Their permitted second read verified the same bytes after ctime changed. The complete tool-output JSON was verified against its persisted SHA-256; the stored 4096-byte prefix alone was insufficient. BOA-12 setup run `b3986762` succeeded at 13:33:24.493. Its warm deadline was 13:38:22.030; drain and suspend completed before Boat's archive completed at 13:44:23.518. Saved continuation `b7b60bc3` started at 13:45:27.509, kept its original native/logical/provider identities, and succeeded at 13:51:17.289 without recovery. Its final warm deadline was 13:56:15.423. Drain and suspend completed before final natural archive at **13:56:40.667**, independently observed archived/healthy at 13:58:37.440. No viewer, guest probe, forced stop or deployment occurred during either idle interval. [Complete source-bound control and lifecycle receipt](assets/2026-10-10-boat/evidence/staging29-independent-file-control.json). -The reviewed packaging correction `b64d630969` creates independent regular files only in a new unpublished pack, preserves bytes/modes/symlinks, and binds the layout policy into the package digest. Existing immutable packs and saved identities are preserved. This is now integrated into the staging candidate, but **has not yet been deployed or qualified with Claude**. +The reviewed packaging correction `b64d630969` creates independent regular files only in a new unpublished pack, preserves bytes/modes/symlinks, and binds the layout policy into the package digest. Existing immutable packs and saved identities are preserved. This is integrated into the staging candidate; deployment30 is running, but the source is **not yet verified serving or qualified with Claude**. -A separate saved task, BOA-2, failed admission before the control script ran: `runner_remote_retirement_checkpoint_unsettled: controller_commands`. Its prior lease had been retained and then released before the controller's drain/suspend completed. The checkpoint correctly refused unsafe recovery. Reviewed core fix `e5b70f39c4` requires exact warm-session closure before physical retirement for stopping dispositions, preserves failed-close fences, and prevents stale retention bookkeeping from overwriting release. The source of the competing release is still under investigation; no historical pending command or quarantine was cleared. A healthy-session prevention test will not count as recovery of this historical failed session. +A separate saved task, BOA-2, failed admission before the control script ran: `runner_remote_retirement_checkpoint_unsettled: controller_commands`. Its prior lease had been retained and then released before the controller's drain/suspend completed. The checkpoint correctly refused unsafe recovery. Reviewed core fix `e5b70f39c4` requires exact warm-session closure before physical retirement for stopping dispositions, preserves failed-close fences, and prevents stale retention bookkeeping from overwriting release. Recovery callback regressions independently reproduced two premature-stop cases. Core `249097811f` preserves the active executor’s cleanup ownership for a fully validated saved computer run; after restart, it uses the frozen successful-run lifecycle without extending the existing warm deadline. All 31 recovery cases, typecheck, boundary checks and independent review passed. Sandbox copy-back handoff is unchanged. The historical second-stop caller remains inferred from source, not identified by a live stack trace. No historical pending command or quarantine was cleared. A healthy-session prevention test will not count as recovery of this historical failed session. [Review and regression receipts](assets/2026-10-10-boat/evidence/idle-fixes-verification.json). | Surface | Source and evidence | | --- | --- | -| Combined implementation | `7d377bb93af8cb43241e8ed17d456289267a1d16` contains core `3223e57b74`. At 13:36 UTC both exact heads had 53 successful checks, review 5/5 and no unresolved threads; four earlier UI checks were superseded/cancelled. New runtime/shutdown integration is in progress. | -| Runtime prerequisite | `b64d6309693e7c10ee783dbd3313ae3a007cf5c1` is pushed with independent-file packaging and failure-only link-count diagnostics. Focused packaging tests, runner/server typechecks and independent review passed; new-head CI is pending. | -| Staging candidate | `a68163746139b91d54750b104254fe40da527f12` integrates reviewed runtime, credential/workspace, Kubernetes-route and packaging corrections. Packaging integration passed 206 tests, six existing platform skips, runner/server typechecks and independent review. Shutdown integration and deployment are pending. All staging migration history is preserved. | +| Combined implementation | `e077c3fbb717520f107068d450587eddacf3c1c8` includes reviewed core `249097811f` and runtime `b64d630969`. Full workspace typecheck and build passed. [Source-bound receipt](assets/2026-10-10-boat/evidence/stack-e077-typecheck-build.json). Core `249097811f` is pushed with 99 changed files and fresh CI/review pending. Previous exact core/UI heads each had 53 successful checks and review 5/5; those checks do not certify the new head. | +| Runtime prerequisite | `b64d6309693e7c10ee783dbd3313ae3a007cf5c1` is pushed with independent-file packaging and failure-only link-count diagnostics. Focused packaging tests, runner/server typechecks and independent review passed. Exact-head CI is complete: 53 successful checks, two expected skips, review 5/5 and no unresolved threads. | +| Staging candidate | `c2861407ff2e2171df0f7c79ad2a92f8c2c21c6b` integrates reviewed runtime, credential/workspace, Kubernetes-route, packaging, shutdown and recovery corrections. Packaging integration passed 206 tests, six existing platform skips and typechecks. Shutdown integration passed 165 focused tests and server typecheck; recovery integration passed 31 tests. Its final server typecheck passed; deployment30 is running through the targeted workflow. All 338 migration blobs match the serving source exactly. | | Verified staging source | Deployment29 remains `05f3725a`. Claude cold/warm passed; its post-idle continuation failed with module link-count rejection. The later controlled file experiment above passed without changing this deployment. Deployment27 `2605a03f` passed saved Codex execution, agent computer use, generated preview, state-preserving warm HMR and natural process cleanup. These earlier-source journeys do not qualify the pending image. | | Local qualification | Local51 passed cold/warm/archive/saved continuation/final archive with the same Node, sidecar and lock bytes as staging28, but a different package closure. No claim extends this pass to the new packaging policy. | | Carried evidence | Deployment26 passed simultaneous legacy admission and cleanup. Deployment25 passed embedded human mouse/keyboard control in Chrome and exact Codex child cancellation while its Claude sibling survived and completed. Earlier local checks cover file editing, detach/reconnect and company boundaries. Crash recovery remains unverified. | diff --git a/doc/plans/assets/2026-10-10-boat/evidence/idle-fixes-verification.json b/doc/plans/assets/2026-10-10-boat/evidence/idle-fixes-verification.json new file mode 100644 index 00000000000..73f0d0d5104 --- /dev/null +++ b/doc/plans/assets/2026-10-10-boat/evidence/idle-fixes-verification.json @@ -0,0 +1,179 @@ +{ + "schema": "paperclip/boat-reviewed-idle-fixes/v1", + "combinedSource": "e077c3fbb717520f107068d450587eddacf3c1c8", + "shutdown": { + "schema": "paperclip.boat.warm-stop-ordering-verification.v1", + "baseCommit": "3223e57b747844834d57d3915d8e7c60b119c723", + "scope": "Explicit heartbeat stopping dispositions close the exact warm native session before computer retirement; no reaper or recovery-policy changes.", + "evidence": { + "observed": "Same lease retained at13:10:04.489 then stop_and_retain released at13:10:06.365; owner subsequently retired before idle close.", + "unproven": "The activity log does not identify the second release caller; recovery missing-sync-reference fallback is a source hypothesis, not a live stack trace." + }, + "validation": { + "baselineFailingRegressions": 11, + "baselineLogs": [ + "warm-stop-order-red.log", + "warm-stop-races-red.log", + "warm-stop-retention-red.log", + "warm-stop-inflight-red.log", + "warm-stop-idle-red.log" + ], + "fullAffectedSuites": { + "files": 3, + "passed": 1053, + "skipped": 0, + "log": "warm-stop-final-suites.log", + "boundary": "Before final once-only pending-close flag; final delta separately qualified." + }, + "finalDelta": { + "passed": 9, + "filtered": 833, + "log": "warm-stop-final-delta.log", + "boundary": "Final source including idle-timeout late request and all three failure fences." + }, + "serverTypecheck": { + "exitCode": 0, + "log": "warm-stop-final-tsc.log" + }, + "diffCheck": "passed", + "database": "Disposable boat_wait_test_UUID databases on task-owned PostgreSQL; product database untouched.", + "priorAttemptLimits": [ + "Initial sandbox DB attempt blocked by EPERM; authorized isolated-DB rerun reproduced two real failures.", + "Intermediate new useRunnerd fixture reused an on-disk identity; fixed test IDs, no product fence changed." + ], + "finalExecutorSuite": { + "passed": 841, + "skipped": 0, + "log": "warm-stop-executor-final.log", + "boundary": "After first two late-explicit-request cases, before final one-line idle catch guard." + }, + "coreChangedFileCount": 98, + "workingTree": "clean; commit not pushed" + }, + "invariants": [ + "keep_running does not request close", + "Every stopping disposition defers physical release while exact closure is busy or failed; existing failed remote destroy exception remains destroy-only", + "Exact run and owner-generation retirement fences remain", + "Stop during retention or after retention drains once at ownership release", + "Explicit close failure stays registered even without instruction working copy", + "Late warm bookkeeping cannot overwrite released or rebound lease metadata/termination receipt", + "Pending controller command and checkpoint guards unchanged" + ], + "liveQualification": "Not deployed or tested live", + "review": "Independent peer approved final source and regression coverage; no remaining findings.", + "commit": "e5b70f39c42f0f5cb8e374f751074f4245724fa3" + }, + "recovery": { + "schema": "paperclip.boat.recovery-warm-policy-verification.v1", + "sourceCommit": "249097811f95d4578e6dd2457035ebae46c9a0b8", + "parentCommit": "fb92e12c226b9ae2d80a1518832334e52dec6dd0", + "scope": [ + "server/src/services/heartbeat/recovery.ts", + "server/src/services/heartbeat/recovery.test.ts" + ], + "coreChangedFiles": 99, + "baseline": { + "testsFailed": 2, + "testsPassed": 1, + "log": "recovery-warm-policy-red.log", + "failures": [ + "Active computer executor cleanup received a competing stop_and_retain request", + "Inactive frozen warm computer policy defaulted to stop_and_retain" + ], + "historicalCallerAttribution": "The staging keep_running then stop_and_retain receipt proves conflicting release dispositions, but does not identify the original callback caller. The recovery path was independently reproduced." + }, + "verification": { + "recoverySuite": { + "passed": 31, + "failed": 0, + "skipped": 0, + "log": "recovery-warm-policy-green.log" + }, + "serverTypecheck": { + "exitCode": 0, + "log": "recovery-warm-policy-tsc.log" + }, + "moduleBoundaryGate": { + "exitCode": 0, + "log": "recovery-warm-policy-boundaries.log" + }, + "diffCheck": "passed", + "independentReview": "approved by live_state_probe; no remaining actionable findings" + }, + "invariants": [ + "Active guard requires native runtime, exact run/company/agent/task binding in fully parsed frozen execution input, valid persisted computer descriptor, and no copy-back/export handoff", + "Inactive successful frozen warm policy keeps the existing disposition without invoking retainWarm or changing a computer-owner deadline", + "Per-turn, malformed, missing, mismatched, failed, cancelled, noncomputer and export-retry cases remain conservative", + "Sandbox copy-back callbacks still release and finalize while an executor remains active", + "Existing exact-owner shutdown ordering, pending controller command checks, checkpoint fencing and lease retention CAS remain unchanged" + ], + "testBoundary": "Actual startup and periodic recovery entrypoints invoke the production callback via a mocked native reconciler. The active test uses a real isolated database lease and the real main-path active-to-retained operation; 60s orphan-sweep backoff isolates the callback from unrelated ownerless fixture orphan cleanup.", + "runtimeChanges": "No provider, guest, product database, staging deployment or owner deadline changes were made during verification.", + "liveAcceptance": "Not yet exercised on deployed source; this receipt makes no saved-session recovery claim." + }, + "integration": { + "capturedAt": "2026-10-11T14:03:48.343035+00:00", + "core": { + "commit": "fb92e12c226b9ae2d80a1518832334e52dec6dd0", + "shutdownParent": "e5b70f39c42f0f5cb8e374f751074f4245724fa3", + "prerequisiteParent": "b64d6309693e7c10ee783dbd3313ae3a007cf5c1", + "pathsAgainstPrerequisite": 98, + "tests": { + "passed": 165, + "filtered": 903, + "log": "core-layout-shutdown-integration-tests.log" + }, + "serverTypecheck": { + "exitCode": 0, + "log": "core-layout-shutdown-integration-tsc.log" + } + }, + "staging": { + "commit": "46cbfc67a3bd317ba2844a557b26c8b8fda080b9", + "packagingParent": "a68163746139b91d54750b104254fe40da527f12", + "shutdownCherryPick": "e5b70f39c42f0f5cb8e374f751074f4245724fa3", + "migrationHistoryPreserved": true, + "initialAttempt": { + "passed": 148, + "failed": 15, + "cause": "Embedded PostgreSQL bootstrap failed before database assertions", + "log": "staging-layout-shutdown-integration-tests.log" + }, + "isolatedRerun": { + "passed": 165, + "filtered": 900, + "log": "staging-layout-shutdown-isolated-tests.log", + "databasePolicy": "Only uniquely named disposable boat_wait_test_UUID databases on task-owned PostgreSQL; product database untouched" + }, + "serverTypecheck": { + "exitCode": 0, + "log": "staging-layout-shutdown-integration-tsc.log" + } + }, + "resolution": "Preserved package_version_mismatch verifier before serialized file-layout validation; retained-aware manifest, root-alias, active-provider compatibility, exact digest, and shutdown fences unchanged.", + "review": { + "reviewer": "live_state_probe", + "verdict": "approved, no actionable findings", + "sourceReviewedCommit": "f647f2101c80795005d7a55f95f03f669a26538f", + "finalCommitHasIdenticalTree": true + }, + "publication": { + "runtimePrerequisitePushed": "b64d6309693e7c10ee783dbd3313ae3a007cf5c1", + "corePushed": false, + "stagingPushed": false, + "deployed": false + }, + "limits": "Focused source integration proof only; new production pack and shutdown lifecycle remain for live qualification. Old immutable packs and saved sessions were not changed." + }, + "stagingMigrationCheck": { + "candidate": "c2861407ff2e2171df0f7c79ad2a92f8c2c21c6b", + "baseline": "05f3725af838b5a074707df1d6ed1e193bdfe49e", + "migrationBlobsUnchanged": 338, + "workingTreeClean": true + }, + "limits": [ + "The historical second-stop caller is inferred from source; the dangerous callback behavior is independently reproduced, not attributed by a live stack trace.", + "Historical quarantined sessions remain unchanged. Healthy-session prevention does not prove recovery of old failures.", + "Full new-package live acceptance is pending." + ] +} diff --git a/doc/plans/assets/2026-10-10-boat/evidence/stack-e077-typecheck-build.json b/doc/plans/assets/2026-10-10-boat/evidence/stack-e077-typecheck-build.json new file mode 100644 index 00000000000..084d917d2fc --- /dev/null +++ b/doc/plans/assets/2026-10-10-boat/evidence/stack-e077-typecheck-build.json @@ -0,0 +1,32 @@ +{ + "observedAt": "2026-10-11T14:12:35.019483+00:00", + "combinedSource": "e077c3fbb717520f107068d450587eddacf3c1c8", + "fullWorkspaceTypecheck": { + "command": "pnpm -r typecheck", + "exitCode": 0, + "logSha256": "84a8567ebc1b9e1ea517c2bbc46fc6acecccf3f8cd8926669997b11a0e1b0d0b" + }, + "fullBuild": { + "command": "pnpm build", + "exitCode": 0, + "logSha256": "906eb42b0c6df32e141bbc535b6b7b173cefaa7dfb6acc388480a9387c91a504" + }, + "stagingSource": "c2861407ff2e2171df0f7c79ad2a92f8c2c21c6b", + "stagingRecoveryTests": { + "passed": 31, + "skipped": 0, + "logSha256": "33d0ed0062851492e5b59c61d1175b87ba3c683752e502575ee5548ba2349a34" + }, + "stagingServerTypecheck": { + "exitCode": 0, + "logSha256": "e855cc9c6f45b0a21965a89b6a14605d032575bb82f4482b99d49ac25ec0d5d0", + "initialAttempt": "Sandbox registry access prevented pnpm signature verification; original failure retained. Normal authorized network rerun passed with no bypass or source change." + }, + "migrationHistory": { + "candidate": "c2861407ff2e2171df0f7c79ad2a92f8c2c21c6b", + "baseline": "05f3725af838b5a074707df1d6ed1e193bdfe49e", + "migrationBlobsUnchanged": 338, + "workingTreeClean": true + }, + "limit": "Historical full local test command not rerun. Fresh exact-head CI remains pending for new core/UI; no full live package acceptance follows from these build checks." +}