diff --git a/package.json b/package.json index b6a185b015b..2213c21acb7 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:notion-generic-live": "node scripts/smoke/notion-generic-live.mjs", "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs scripts/scale-ramp/scale-ramp.test.ts scripts/scale-ramp/scale-ramp-spend.test.ts", + "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/ci/stalled-file-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs scripts/scale-ramp/scale-ramp.test.ts scripts/scale-ramp/scale-ramp-spend.test.ts", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/ci/stalled-file-reporter.mjs b/scripts/ci/stalled-file-reporter.mjs new file mode 100644 index 00000000000..82412feb5c8 --- /dev/null +++ b/scripts/ci/stalled-file-reporter.mjs @@ -0,0 +1,146 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +// A test file that stops making progress otherwise holds its CI job silently +// until the job's time limit cancels it. Turbo prints a task's output only +// when the task ends, so a cancelled job never names the stuck file (server +// shard 2/12 on 2026-11-16: five cancellations at 20m16s, no file named). +// +// This reporter watches every running file. When one starts no test, hook or +// result for the stall limit, it prints a GitHub error annotation naming the +// file and the step it was in, then ends the run with a failure, so the job +// fails minutes earlier with the file and step in its log. +export const stallEnvName = "PAPERCLIP_VITEST_FILE_STALL_MS"; +// The slowest healthy server files ran about 5 minutes in total +// (issue-recovery-live-run-agent-census, 235-300 s in PR shards), and no +// single test or hook in a healthy run goes this long without progress. +export const defaultStallMs = 8 * 60_000; +// How long a cancelled run may take to stop before the process exits anyway. +export const exitGraceMs = 30_000; +const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); + +export function stallLimitMs(env = process.env) { + const configured = env[stallEnvName]?.trim(); + if (!configured) return defaultStallMs; + const parsed = Number(configured); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${stallEnvName} must be a positive integer of milliseconds. Received "${configured}".`); + } + return parsed; +} + +// Watching is for CI, where a stuck file costs a whole job; a local run can be +// stopped by hand. The variable turns it on anywhere, for example to reproduce. +export function stalledFileReporterEnabled(env = process.env) { + return env.GITHUB_ACTIONS === "true" || Boolean(env[stallEnvName]?.trim()); +} + +function escapeData(value) { + return String(value).replaceAll("%", "%25").replaceAll("\r", "%0D").replaceAll("\n", "%0A"); +} + +function escapeProperty(value) { + return escapeData(value).replaceAll(":", "%3A").replaceAll(",", "%2C"); +} + +export function stallAnnotation(entry, now) { + const minutes = ((now - entry.progressAt) / 60_000).toFixed(1); + const properties = [`file=${escapeProperty(entry.file)}`, `title=${escapeProperty("Test file stopped making progress")}`]; + return `::error ${properties.join(",")}::${escapeData( + `${entry.file} made no progress for ${minutes} min while in: ${entry.step}. ` + + "The run was stopped so this job fails with the file's name instead of reaching its time limit.")}`; +} + +export default class StalledFileReporter { + constructor(options = {}) { + this.limitMs = options.limitMs ?? stallLimitMs(); + this.now = options.now ?? Date.now; + this.exit = options.exit ?? (code => process.exit(code)); + this.write = options.write ?? (line => process.stderr.write(`${line}\n`)); + this.root = options.root ?? repoRoot; + this.graceMs = options.graceMs ?? exitGraceMs; + this.intervalMs = options.intervalMs ?? Math.min(15_000, Math.max(250, Math.floor(this.limitMs / 4))); + this.files = new Map(); + this.timer = null; + this.stopped = false; + } + + onInit(vitest) { + this.vitest = vitest; + } + + onTestRunStart() { + this.stopped = false; + if (this.timer) return; + // The reporter runs in the main Vitest process, whose timers tests never + // fake. The interval must not keep a finished run alive. + this.timer = setInterval(() => this.check(), this.intervalMs); + this.timer.unref?.(); + } + + progress(moduleId, step) { + const entry = this.files.get(moduleId); + if (!entry) return; + entry.progressAt = this.now(); + entry.step = step; + } + + // Queued fires before the file is imported, so a file that never finishes + // loading (a top-level await or a blocking setup import) is watched too. + onTestModuleQueued(testModule) { + const file = path.relative(this.root, testModule.moduleId).split(path.sep).join("/"); + this.files.set(testModule.moduleId, { file, progressAt: this.now(), step: "loading the file" }); + } + + onTestModuleStart(testModule) { + if (!this.files.has(testModule.moduleId)) this.onTestModuleQueued(testModule); + this.progress(testModule.moduleId, "starting the file's tests"); + } + + onHookStart(hook) { + const moduleId = hook.entity?.module?.moduleId ?? hook.entity?.moduleId; + const owner = hook.entity?.fullName ?? hook.entity?.name; + this.progress(moduleId, owner ? `${hook.name} of ${owner}` : hook.name); + } + + onHookEnd(hook) { + this.onHookStart({ ...hook, name: `after ${hook.name}` }); + } + + onTestCaseReady(testCase) { + this.progress(testCase.module.moduleId, `test "${testCase.fullName}"`); + } + + onTestCaseResult(testCase) { + this.progress(testCase.module.moduleId, `after test "${testCase.fullName}"`); + } + + onTestModuleEnd(testModule) { + this.files.delete(testModule.moduleId); + } + + onTestRunEnd() { + this.files.clear(); + if (this.timer) clearInterval(this.timer); + this.timer = null; + // A cancelled run can report its finished files as passed. It still failed: + // exit once the other reporters have printed their summaries. + if (this.stopped) setTimeout(() => this.exit(1), 1000); + } + + check() { + if (this.stopped) return; + const now = this.now(); + const stalled = [...this.files.values()].filter(entry => now - entry.progressAt >= this.limitMs); + if (stalled.length === 0) return; + this.stopped = true; + for (const entry of stalled) this.write(stallAnnotation(entry, now)); + const running = [...this.files.values()].map(entry => entry.file).sort(); + this.write(`[stalled-file-reporter] stopping the run; files still running: ${running.join(", ")}`); + process.exitCode = 1; + // A stalled worker may never answer a cancel, so exit after a grace period + // either way. This timer is kept referenced on purpose. + setTimeout(() => this.exit(1), this.graceMs); + void Promise.resolve(this.vitest?.cancelCurrentRun("stalled-file")).catch(() => undefined); + } +} diff --git a/scripts/ci/stalled-file-reporter.test.mjs b/scripts/ci/stalled-file-reporter.test.mjs new file mode 100644 index 00000000000..840cf6d5e6f --- /dev/null +++ b/scripts/ci/stalled-file-reporter.test.mjs @@ -0,0 +1,114 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import StalledFileReporter, { defaultStallMs, stallAnnotation, stalledFileReporterEnabled, stallLimitMs } + from "./stalled-file-reporter.mjs"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); +const reporterPath = fileURLToPath(new URL("./stalled-file-reporter.mjs", import.meta.url)); + +test("the stall limit defaults to eight minutes and accepts a positive override", () => { + assert.equal(stallLimitMs({}), defaultStallMs); + assert.equal(defaultStallMs, 8 * 60_000); + assert.equal(stallLimitMs({ PAPERCLIP_VITEST_FILE_STALL_MS: " 1500 " }), 1500); + assert.equal(stallLimitMs({ PAPERCLIP_VITEST_FILE_STALL_MS: "" }), defaultStallMs); + for (const value of ["0", "-5", "1.5", "soon"]) { + assert.throws(() => stallLimitMs({ PAPERCLIP_VITEST_FILE_STALL_MS: value }), /positive integer/, value); + } +}); + +test("the reporter watches in GitHub Actions, or wherever the limit is set", () => { + assert.equal(stalledFileReporterEnabled({}), false); + assert.equal(stalledFileReporterEnabled({ GITHUB_ACTIONS: "true" }), true); + assert.equal(stalledFileReporterEnabled({ PAPERCLIP_VITEST_FILE_STALL_MS: "1000" }), true); +}); + +test("a file stops the run only once it makes no progress for the whole limit", () => { + let now = 0; + const lines = [], exits = []; + const cancels = []; + const reporter = new StalledFileReporter({ limitMs: 1000, now: () => now, write: line => lines.push(line), + exit: code => exits.push(code), root: "/repo", graceMs: 0 }); + reporter.onInit({ cancelCurrentRun: reason => { cancels.push(reason); return Promise.resolve(); } }); + const moduleId = "/repo/server/src/stuck.test.ts"; + const testCase = { fullName: "suite > waits forever", module: { moduleId } }; + reporter.onTestModuleStart({ moduleId }); + reporter.onTestModuleStart({ moduleId: "/repo/server/src/healthy.test.ts" }); + now = 900; reporter.onTestCaseReady(testCase); + now = 1800; reporter.onTestCaseResult({ fullName: "ok", module: { moduleId: "/repo/server/src/healthy.test.ts" } }); + reporter.check(); + assert.deepEqual(lines, [], "progress 900 ms ago is not a stall"); + now = 1900; reporter.check(); + assert.equal(lines.length, 2); + assert.match(lines[0], /^::error file=server\/src\/stuck\.test\.ts,title=Test file stopped making progress::/); + assert.match(lines[0], /while in: test "suite > waits forever"/); + assert.equal(lines[1], "[stalled-file-reporter] stopping the run; files still running: server/src/healthy.test.ts, server/src/stuck.test.ts"); + assert.deepEqual(cancels, ["stalled-file"]); + reporter.check(); + assert.equal(lines.length, 2, "a stopped run reports once"); + process.exitCode = 0; +}); + +test("finished files and hooks count as progress", () => { + let now = 0; + const lines = []; + const reporter = new StalledFileReporter({ limitMs: 1000, now: () => now, write: line => lines.push(line), exit: () => {}, root: "/repo" }); + const moduleId = "/repo/a.test.ts"; + reporter.onTestModuleStart({ moduleId }); + now = 800; reporter.onHookStart({ name: "beforeAll", entity: { moduleId } }); + now = 1600; reporter.check(); + assert.deepEqual(lines, []); + reporter.onTestModuleEnd({ moduleId }); + now = 10_000; reporter.check(); + assert.deepEqual(lines, []); + assert.equal(stallAnnotation({ file: "a,b:c.test.ts", progressAt: 0, step: "x\ny" }, 90_000), + "::error file=a%2Cb%3Ac.test.ts,title=Test file stopped making progress::a,b:c.test.ts made no progress for 1.5 min while in: x%0Ay." + + " The run was stopped so this job fails with the file's name instead of reaching its time limit."); +}); + +test("a file that never finishes loading is named from the moment it is queued", () => { + let now = 0; + const lines = []; + const reporter = new StalledFileReporter({ limitMs: 1000, now: () => now, write: line => lines.push(line), exit: () => {}, root: "/repo" }); + reporter.onTestModuleQueued({ moduleId: "/repo/server/src/top-level-await.test.ts" }); + now = 1000; reporter.check(); + assert.match(lines[0] ?? "", /^::error file=server\/src\/top-level-await\.test\.ts,.*while in: loading the file\./); + process.exitCode = 0; +}); + +// The failure this guards: a test awaiting a promise that never settles, with +// a test timeout longer than the CI job, holds Vitest until the job is +// cancelled and never prints the file's name. +for (const [label, body, step] of [ + ["a test that never settles", 'import { test } from "vitest";\ntest("waits forever", () => new Promise(() => {}), 600_000);\n', 'test "waits forever"'], + ["a top-level await that never settles", 'import { test } from "vitest";\nawait new Promise(() => {});\ntest("never collected", () => {});\n', "loading the file"], +]) test(`a real Vitest run with ${label} fails fast and names the file`, () => { + const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "stalled-file-reporter-"))); + try { + writeFileSync(path.join(directory, "stuck.test.mjs"), body); + writeFileSync(path.join(directory, "quick.test.mjs"), `import { test } from "vitest"; +test("passes", () => {}); +`); + const configFile = path.join(directory, "vitest.config.mjs"); + writeFileSync(configFile, `export default ${JSON.stringify({ test: { include: ["*.test.mjs"], + reporters: ["default", [reporterPath, { limitMs: 1500, graceMs: 5000, root: directory }]] } })};`); + const started = Date.now(); + const result = spawnSync(process.execPath, [path.join(root, "node_modules/vitest/vitest.mjs"), "run", + "--config", configFile, "--root", directory], { + cwd: directory, encoding: "utf8", timeout: 60_000, + env: { ...process.env, GITHUB_ACTIONS: "", PAPERCLIP_VITEST_FILE_STALL_MS: "" }, + }); + const output = `${result.stdout}\n${result.stderr}`; + assert.equal(result.signal, null, `Vitest was still running after 60 s:\n${output}`); + assert.notEqual(result.status, 0, output); + assert.ok(Date.now() - started < 30_000, `took ${Date.now() - started} ms:\n${output}`); + assert.match(output, /::error file=stuck\.test\.mjs,title=Test file stopped making progress::stuck\.test\.mjs made no progress/); + assert.ok(output.includes(`while in: ${step}.`), output); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts b/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts index 94032f7f2e8..1d0a90dd36a 100644 --- a/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts +++ b/server/src/__tests__/issue-recovery-live-run-agent-census.test.ts @@ -11,9 +11,23 @@ import { seedRecoveryPopulation } from "./helpers/recovery-population.js"; vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1,assigned-work-invariant:1"; }); const MINUTE = 60_000; +const TEST_TIMEOUT_MS = 240_000; +const STEP_TIMEOUT_MS = 20_000; const TOPICS = new Set(["stranded-assignments", "assigned-work-invariant"]); +const realSetTimeout = globalThis.setTimeout; +const realClearTimeout = globalThis.clearTimeout; type Recorded = { at: number; topic: string; name: string; fields: Record }; +/** Bounds one step with the real clock, captured before the timers are faked, + * so a wait that cannot finish fails in seconds and names what it waited for. */ +function bounded(work: Promise, waitingFor: () => string): Promise { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = realSetTimeout(() => reject(new Error(`Timed out after ${STEP_TIMEOUT_MS} ms waiting for ${waitingFor()}`)), STEP_TIMEOUT_MS); + }); + return Promise.race([work, expired]).finally(() => realClearTimeout(timer)); +} + /** * Staging shape: about 860 settled assigned tasks, most of them held by the * same few agents, and one agent whose native run stays live for the whole @@ -27,9 +41,14 @@ describeIssueRecovery("stranded-task checks beside one long live run (census)", const until = Date.now() + ms; let nextTick = Date.now(); while (Date.now() < until) { - if (onTick && Date.now() >= nextTick) { await onTick(); nextTick = Date.now() + 10_000; } - await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); - await Promise.allSettled([...state.passes]); await immediate(); + const at = new Date(Date.now()).toISOString(); + if (onTick && Date.now() >= nextTick) { + await bounded(onTick(), () => `the run event at simulated ${at} to commit`); + nextTick = Date.now() + 10_000; + } + await bounded(vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())), () => `simulated time to advance past ${at}`); + await bounded(Promise.allSettled([...state.passes]), () => `${state.passes.size} delivery passes started by simulated ${at} to settle`); + await immediate(); } } @@ -56,38 +75,52 @@ describeIssueRecovery("stranded-task checks beside one long live run (census)", async function seed() { const f = await state.fixture(); // 12 agents leave 3 workers, so each holds about 266 of the settled reviews. - const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 12, inReview: 800, inProgress: 30, todo: 30 }); + // No agent takes on-demand wakes: a check that repairs a task would start a + // real run while setTimeout is faked, and that run can wait forever. The + // census counts queue checks, not runs. + const population = await seedRecoveryPopulation(state.db, f.companyId, + { agents: 12, inReview: 800, inProgress: 30, todo: 30, wakeOnDemand: false }); + // Settled long before the window, as on staging. Rows stamped now count as + // changed for the first backstop after the warm-up, which would re-check + // every seeded task once. + const settledAt = new Date(Date.now() - 2 * 60 * MINUTE); + await state.db.update(issues).set({ createdAt: settledAt, updatedAt: settledAt }).where(eq(issues.companyId, f.companyId)); + await state.db.update(heartbeatRuns).set({ updatedAt: settledAt }).where(eq(heartbeatRuns.companyId, f.companyId)); + await state.db.update(agents).set({ createdAt: settledAt, updatedAt: settledAt }).where(eq(agents.companyId, f.companyId)); const liveAgentId = population.agentIds[7]!; - const [liveTask] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Live native task", status: "in_progress", + // A settled review until its run starts. An in-progress task with no run + // is real assigned work, so recovery would start a real run for it while + // setTimeout is faked, and that run can wait forever. + const [liveTask] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Live native task", status: "in_review", assigneeAgentId: liveAgentId }).returning({ id: issues.id }); // A paused agent with one waiting task, both settled long before the window. - const old = new Date(Date.now() - 2 * 60 * MINUTE); + // It takes no on-demand wakes, so resuming it never starts a real run. const [held] = await state.db.insert(agents).values({ companyId: f.companyId, name: "Held worker", role: "engineer", status: "paused", - lifecycleHolds: ["manual"], adapterType: "process", reportsTo: population.agentIds[1], createdAt: old, updatedAt: old, + lifecycleHolds: ["manual"], adapterType: "process", reportsTo: population.agentIds[1], createdAt: settledAt, updatedAt: settledAt, adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, - runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } } }).returning({ id: agents.id }); + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false } } }).returning({ id: agents.id }); const [waiting] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Waits for its agent", status: "todo", - assigneeAgentId: held!.id, createdAt: old, updatedAt: old }).returning({ id: issues.id }); + assigneeAgentId: held!.id, createdAt: settledAt, updatedAt: settledAt }).returning({ id: issues.id }); return { ...f, population, liveAgentId, liveTaskId: liveTask!.id, heldAgentId: held!.id, waitingTaskId: waiting!.id }; } async function boot() { - const f = await seed(); + const f = await bounded(seed(), () => "the fixture to be seeded"); const events = record(); vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); - await state.start({}); + await bounded(state.start({}), () => "the recovery workers to start"); await advance(10 * MINUTE); events.length = 0; return { ...f, events }; } - /** What execution start writes (heartbeat.ts): the run, then the agent's - * status flip to running without a heartbeat time. */ + /** What execution start writes (heartbeat.ts): the run, the task's checkout, + * then the agent's status flip to running without a heartbeat time. */ async function startLiveRun(f: Awaited>) { const [run] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.liveAgentId, status: "running", runtimeMode: "native", startedAt: new Date(), contextSnapshot: { issueId: f.liveTaskId }, nativeIssueId: f.liveTaskId }) .returning({ id: heartbeatRuns.id }); - await state.db.update(issues).set({ executionRunId: run!.id }).where(eq(issues.id, f.liveTaskId)); + await state.db.update(issues).set({ status: "in_progress", executionRunId: run!.id }).where(eq(issues.id, f.liveTaskId)); await state.db.update(agents).set({ status: "running", updatedAt: new Date() }).where(eq(agents.id, f.liveAgentId)); const runId = run!.id; return { @@ -102,7 +135,7 @@ describeIssueRecovery("stranded-task checks beside one long live run (census)", it("does not re-check the live agent's settled tasks while its run stays live", async () => { const f = await boot(); - const live = await startLiveRun(f); + const live = await bounded(startLiveRun(f), () => "the live run to start"); let ticks = 0; const before = state.meter.read(); await advance(30 * MINUTE, () => { ticks += 1; return live.tick(); }); @@ -121,11 +154,11 @@ describeIssueRecovery("stranded-task checks beside one long live run (census)", expect(result.strandedByWake.hint ?? 0).toBeLessThanOrEqual(ticks + 10); expect(result.assignedWork).toBeLessThanOrEqual(10); expect(result.signalsBeyondRunEvents).toBe(0); - }, 1_800_000); + }, TEST_TIMEOUT_MS); it("still checks a task of the live agent within the backstop bound when its own rows change without a hint", async () => { const f = await boot(); - const live = await startLiveRun(f); + const live = await bounded(startLiveRun(f), () => "the live run to start"); await advance(6 * MINUTE, live.tick); f.events.length = 0; const target = f.population.inReview.find((_, index) => index % 3 === 0)!; @@ -133,31 +166,32 @@ describeIssueRecovery("stranded-task checks beside one long live run (census)", expect(current!.assignee).toBe(f.liveAgentId); const changedAt = Date.now(); // A direct write that sends no hint (for example an edit by an older server). - await state.db.update(issues).set({ status: "todo", updatedAt: new Date() }).where(eq(issues.id, target)); + await bounded(state.db.update(issues).set({ status: "todo", updatedAt: new Date() }).where(eq(issues.id, target)), + () => "the direct task write"); await advance(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE, live.tick); const checked = f.events.find(event => event.topic === "stranded-assignments" && event.name === "check" && event.fields.issueId === target); expect(checked).toBeDefined(); expect(checked!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); - }, 1_800_000); + }, TEST_TIMEOUT_MS); it("still checks a resumed agent's task within the backstop bound when the resume sends no hint", async () => { const f = await boot(); - const live = await startLiveRun(f); + const live = await bounded(startLiveRun(f), () => "the live run to start"); await advance(6 * MINUTE, live.tick); f.events.length = 0; const waiting = f.waitingTaskId; const changedAt = Date.now(); - await state.db.update(agents).set({ status: "idle", lifecycleHolds: [], updatedAt: new Date() }) - .where(eq(agents.id, f.heldAgentId)); + await bounded(state.db.update(agents).set({ status: "idle", lifecycleHolds: [], updatedAt: new Date() }) + .where(eq(agents.id, f.heldAgentId)), () => "the agent resume write"); await advance(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE, live.tick); const checked = f.events.find(event => event.topic === "stranded-assignments" && event.name === "check" && event.fields.issueId === waiting); expect(checked).toBeDefined(); expect(checked!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); - }, 1_800_000); + }, TEST_TIMEOUT_MS); it("still re-checks every settled task in the hourly full pass", async () => { const f = await boot(); - const live = await startLiveRun(f); + const live = await bounded(startLiveRun(f), () => "the live run to start"); await advance(ISSUE_RECOVERY_FULL_BACKSTOP_MS + 5 * MINUTE, live.tick); const checked = new Set(checksOf(f.events, "stranded-assignments").map(event => String(event.fields.issueId))); const missing = f.population.inReview.filter(id => !checked.has(id)); @@ -165,5 +199,5 @@ describeIssueRecovery("stranded-task checks beside one long live run (census)", const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.status} = 'running'`)); expect(runs.map(run => run.id)).toContain(live.runId); - }, 1_800_000); + }, TEST_TIMEOUT_MS); }); diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 6d024d3da2b..b1a79860e2e 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -1,6 +1,6 @@ { "inputs": { - "package.json": "b6a185b015b2ec9ffc80564f0970916b65b7d6ba", + "package.json": "2213c21acb7ce340b07aa114c69974a266a0bad8", "packages/plugins/sdk/package.json": "b7a4936e2803fd0d3b32dd7f17fc1ea86609ead0", "packages/plugins/sdk/src/bundlers.ts": "883bf3d9f2bd42877cfc0c3707cd882a6f6248d0", "packages/plugins/sdk/src/define-plugin.ts": "a99e07ce77ef72fcf1df1347346aacb8ba12e284", diff --git a/vitest.config.ts b/vitest.config.ts index 4961fc0eef7..1beb791259a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,5 +1,6 @@ import { configDefaults, defineConfig } from "vitest/config"; import { retryReporterEnabled } from "./scripts/ci/vitest-retry.mjs"; +import { stalledFileReporterEnabled } from "./scripts/ci/stalled-file-reporter.mjs"; export default defineConfig({ test: { @@ -8,7 +9,9 @@ export default defineConfig({ ? ["./scripts/ci/test-timing-reporter.mjs", "./scripts/ci/oversized-test-files.mjs"] : []), ...(process.env.PAPERCLIP_CI_PROFILE === "true" ? ["./scripts/ci-profile/vitest-reporter.mjs"] : []), // PR lanes pass --retry from scripts/run-vitest-stable.mjs; see scripts/ci/vitest-retry.mjs. - ...(retryReporterEnabled() ? ["./scripts/ci/retry-reporter.mjs"] : [])], + ...(retryReporterEnabled() ? ["./scripts/ci/retry-reporter.mjs"] : []), + // Fails a CI run whose file stops making progress, naming the file; see the reporter. + ...(stalledFileReporterEnabled() ? ["./scripts/ci/stalled-file-reporter.mjs"] : [])], projects: [ "packages/shared", "packages/skills-catalog",