Merge pinned master into work folders base and reconcile migration 0277

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/scripts/tests/post-merge-runner-routing.test.mjs

@@ -10,7 +10,7 @@ const base = {
   ref: "refs/heads/master", event_name: "push", sha,
 };
 const expectedJobs = {
-  "cloud-readiness.yml": ["artifacts", "source_verified", "ready"],
+  "cloud-readiness.yml": [],
   "cloud-artifacts.yml": ["dispatch_migrator"],
   "release-verify.yml": ["typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"],
   "runner-chaos-evals.yml": ["chaos_and_recovery"],
@@ -81,3 +81,27 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) {
     });
   }
 }
+
+
+test("Cloud readiness bookkeeping never waits for the AWS verification fleet", () => {
+  const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8");
+  const bodies = new Map();
+  for (const [name, needs] of [
+    ["artifacts", null],
+    ["source_verified", "[verify]"],
+    ["ready", "[verify, image, artifacts]"],
+  ]) {
+    const body = workflow.match(new RegExp(`^  ${name}:\\n([\\s\\S]*?)(?=^  [a-z_]+:|(?![\\s\\S]))`, "m"))?.[1];
+    assert.ok(body, `missing ${name} job`);
+    bodies.set(name, body);
+    assert.match(body, /^    runs-on: ubuntu-latest$/m);
+    assert.doesNotMatch(body, /^ +continue-on-error:|^ +if:.*always\(\)/m);
+    assert.match(body, /^    if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'$/m);
+    assert.match(body, /^ +SOURCE_SHA: \$\{\{ github.sha \}\}$/m);
+    assert.equal(body.match(/^    needs: (.+)$/m)?.[1] ?? null, needs, `${name} prerequisites`);
+  }
+  assert.match(bodies.get("artifacts"), /^        run: node scripts\/cloud-readiness.mjs "\$SOURCE_SHA"$/m);
+  assert.match(bodies.get("source_verified"), /^        run: node --test scripts\/cloud-source-verification.test.mjs$/m);
+  assert.match(bodies.get("source_verified"), /echo "Cloud source verified v1: \$SOURCE_SHA"/);
+  assert.match(bodies.get("ready"), /echo "Cloud deployable v1: \$SOURCE_SHA"/);
+});

.github/workflows/cloud-readiness.yml

@@ -32,7 +32,8 @@ jobs:
   artifacts:
     if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
     name: Wait for exact-source cloud artifacts
-    runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
+    # Bookkeeping must not wait for the AWS builders it observes.
+    runs-on: ubuntu-latest
     timeout-minutes: 35
     permissions:
       contents: read
@@ -55,7 +56,8 @@ jobs:
     name: Cloud source verified v1
     needs: [verify]
     if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
-    runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
+    # Bookkeeping must not wait for the AWS builders it observes.
+    runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
       contents: read
@@ -81,7 +83,8 @@ jobs:
     name: Cloud deployable v1
     needs: [verify, image, artifacts]
     if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
-    runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
+    # Bookkeeping must not wait for the AWS builders it observes.
+    runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
       - name: Record cloud readiness

.github/workflows/docker-cloud.yml

@@ -72,24 +72,6 @@ jobs:
         id: tools-epoch
         run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
 
-      # Each SHA exports its own cache. Import recent first-parent caches so
-      # a late older build cannot overwrite a newer build's cache manifest.
-      # The legacy ref keeps the first builds warm during the transition.
-      - name: Select cloud cache ancestry
-        id: cloud-cache
-        env:
-          CACHE_IMAGE: ghcr.io/${{ github.repository }}
-        run: |
-          set -euo pipefail
-          {
-            echo 'sources<<CACHE_SOURCES'
-            for commit in $(git rev-list --first-parent --max-count=10 HEAD); do
-              echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud-$commit"
-            done
-            echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud"
-            echo 'CACHE_SOURCES'
-          } >> "$GITHUB_OUTPUT"
-
       - name: Setup pnpm
         uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
         with:
@@ -175,6 +157,14 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
 
+      # Mixing several historical manifests missed otherwise reusable native
+      # layers on fresh builders. Import the nearest available complete cache.
+      - name: Select cloud cache ancestry
+        id: cloud-cache
+        env:
+          CACHE_IMAGE: ghcr.io/${{ github.repository }}
+        run: node scripts/select-cloud-cache.mjs
+
       # Deployment tooling reads these labels from the registry to verify an
       # image's schema expectations against a migrator before deploying it,
       # without pulling the image. The server refuses to start when the
@@ -239,7 +229,7 @@ jobs:
           # Same-SHA builds serialize above; different SHAs never share a
           # writable cache ref. Registry layers are content-addressed and
           # shared even when cache manifests have separate tags.
-          cache-from: ${{ steps.cloud-cache.outputs.sources }}
+          cache-from: ${{ steps.cloud-cache.outputs.source }}
           cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max
           tags: ${{ steps.meta-cloud.outputs.tags }}
           labels: ${{ steps.meta-cloud.outputs.labels }}

.github/workflows/docker-runner-check.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
       # Compile the real target, then change source in a disposable context.
-      # Require dependency-layer reuse and changed metadata from the real binary.
+      # A fresh builder must import dependencies and produce changed binary metadata.
       # No registry credentials, external cache, or image publication.
       - name: Verify native build and dependency cache reuse
         run: bash scripts/check-docker-runner-cache.sh