diff --git a/.github/scripts/tests/post-merge-runner-routing.test.mjs b/.github/scripts/tests/post-merge-runner-routing.test.mjs index d31d315c042..7c46aa47931 100644 --- a/.github/scripts/tests/post-merge-runner-routing.test.mjs +++ b/.github/scripts/tests/post-merge-runner-routing.test.mjs @@ -7,7 +7,7 @@ const fleet = "runs-on/fleet=paperclip-post-merge-x64/env=public-ci"; const sha = "a".repeat(40); const base = { repository: "paperclipai/paperclip", repository_id: "1170821064", - ref: "refs/heads/master", event_name: "push", sha, + ref: "refs/heads/master", event_name: "push", sha, run_attempt: 1, }; const expectedJobs = { "cloud-readiness.yml": [], @@ -58,6 +58,11 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) { cases.push({ name: "preview of master", inputs: { channel: "preview" } }); cases.push({ name: "stable release", inputs: { channel: "stable" } }); } + cases.push({ name: "first attempt as a string", github: { run_attempt: "1" }, expected: fleet }); + cases.push({ name: "rerun after runner interruption", github: { run_attempt: 2 } }); + cases.push({ name: "later rerun", github: { run_attempt: 3 } }); + cases.push({ name: "rerun attempt as a string", github: { run_attempt: "2" } }); + cases.push({ name: "missing attempt", github: { run_attempt: undefined } }); for (const { name, github = {}, inputs: overrides = {}, enabled = "true", expected = "ubuntu-latest" } of cases) { test(`${file}/${job}: ${name}`, () => { const context = { github: { ...defaultContext, ...github }, inputs: { ...inputs, ...overrides }, vars: { AWS_POST_MERGE_CI_ENABLED: enabled } }; diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 76128fd1a89..e36bdbccbd2 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -34,7 +34,7 @@ jobs: typecheck: if: ${{ !inputs.runner_binaries_only }} name: Typecheck - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -120,7 +120,7 @@ jobs: general_tests: if: ${{ !inputs.runner_binaries_only }} name: General tests (${{ matrix.group_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -232,7 +232,7 @@ jobs: serialized_tests: if: ${{ !inputs.runner_binaries_only }} name: Serialized tests (${{ matrix.shard_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -282,7 +282,7 @@ jobs: runner_workflow_evals: if: ${{ !inputs.runner_binaries_only }} name: Runner workflow eval scorer contract - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 10 permissions: contents: read @@ -313,7 +313,7 @@ jobs: verify_paperclip_runner: if: ${{ !inputs.runner_binaries_only }} name: Verify Paperclip Runner (${{ matrix.lane }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -523,7 +523,7 @@ jobs: build: if: ${{ !inputs.runner_binaries_only }} name: Build - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a398e30dae..c7b310f4ee1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -94,11 +94,11 @@ jobs: # falls back to GitHub-hosted runners, including raced merge dispatches. name: Check preview artifacts if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && !inputs.dry_run - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} permissions: contents: read # Preserve the previous hosted default; only AWS needs the Fleet limit. - timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 10 || 360 }} + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 10 || 360 }} outputs: image: ${{ steps.plan.outputs.image }} packages: ${{ steps.plan.outputs.packages }} @@ -122,7 +122,7 @@ jobs: name: Build preview migrator needs: plan_preview if: needs.plan_preview.outputs.packages == 'true' - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 30 permissions: contents: read diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml index 91438bec3a8..810d28ec2e7 100644 --- a/.github/workflows/runner-chaos-evals.yml +++ b/.github/workflows/runner-chaos-evals.yml @@ -20,8 +20,8 @@ concurrency: jobs: chaos_and_recovery: name: Restart, replay, trace, and recovery faults - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} - timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} permissions: contents: read diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 3812107da15..a55052001e4 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -382,6 +382,13 @@ config; an explicit shared test database retains one partition per job. The Runner transport lease-expiry and cold-admission regressions live in their own file so those real-clock waits overlap other files in the existing pool. +Exact-source release verification uses the post-merge AWS fleet on the first +attempt. Reruns of its verification and chaos jobs use GitHub-hosted runners. +Exact-source cloud-migrator planning and package builds use the same policy. +This gives interrupted jobs an independent recovery path. It keeps the same +source, commands, assertions and required gates. A failed test still needs a fix; +a successful rerun does not explain an earlier failure. + The two PR Runner Vitest jobs separate package tests from native server tests. The first runs the complete package inventory; the second runs the complete native server group. The existing `--shard=1/2` and `--shard=2/2` commands route diff --git a/server/src/__tests__/heartbeat-workspace-busy.test.ts b/server/src/__tests__/heartbeat-workspace-busy.test.ts index 55649bc9ba8..c01ecb082b1 100644 --- a/server/src/__tests__/heartbeat-workspace-busy.test.ts +++ b/server/src/__tests__/heartbeat-workspace-busy.test.ts @@ -209,33 +209,14 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { await db.delete(companies); } - async function waitForRunToLeaveActiveStates(runId: string, timeoutMs = 10_000) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const run = await heartbeat.getRun(runId); - if (run && !["queued", "running"].includes(run.status)) return run; - await new Promise((resolve) => setTimeout(resolve, 50)); - } - return await heartbeat.getRun(runId); + async function waitForRunToLeaveActiveStates(runId: string) { + await heartbeat.drainActiveRunExecutions(); + return heartbeat.getRun(runId); } - // A deferral does two writes in order: first it cancels the original run, - // then it inserts the scheduled-retry row. waitForRunToLeaveActiveStates - // returns after the first write, so a read of the retry row can land before - // the second write and find nothing. Poll until the retry row exists so the - // retry-row assertions never observe the gap between the two writes. - async function waitForRetryRun(originalRunId: string, timeoutMs = 10_000) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const retryRun = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.retryOfRunId, originalRunId)) - .then((rows) => rows[0] ?? null); - if (retryRun) return retryRun; - await new Promise((resolve) => setTimeout(resolve, 50)); - } - return await db + async function waitForRetryRun(originalRunId: string) { + await heartbeat.drainActiveRunExecutions(); + return db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.retryOfRunId, originalRunId)) diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 971ec4f4d0b..8a1d010f8a7 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -70,11 +70,11 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace if (external) await db.execute(sql`truncate table environments restart identity cascade`); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { - await db?.$client.end(); + await db?.$client.end({ timeout: 1 }); await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); - }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); beforeEach(async () => { persistenceBefore = new Map(persistenceKeys.map(key => [key, process.env[key]])); vi.restoreAllMocks();